sqlmesh.utils.date API documentation

Edit on GitHub

# sqlmesh.utils.date

View Source

      1from __future__ import annotations
      2
      3import time
      4import typing as t
      5import warnings
      6
      7warnings.filterwarnings(
      8    "ignore",
      9    message="The localize method is no longer necessary, as this time zone supports the fold attribute",
     10)
     11from datetime import date, datetime, timedelta, timezone
     12
     13import dateparser
     14from sqlglot import exp
     15
     16UTC = timezone.utc
     17TimeLike = t.Union[date, datetime, str, int, float]
     18MILLIS_THRESHOLD = time.time() + 100 * 365 * 24 * 3600
     19DATE_INT_FMT = "%Y%m%d"
     20
     21if t.TYPE_CHECKING:
     22    from sqlmesh.core.scheduler import Interval
     23
     24
     25def now() -> datetime:
     26    """
     27    Current utc datetime.
     28
     29    Returns:
     30        A datetime object with tz utc.
     31    """
     32    return datetime.utcnow().replace(tzinfo=UTC)
     33
     34
     35def now_timestamp() -> int:
     36    """
     37    Current utc timestamp.
     38
     39    Returns:
     40        UTC epoch millis timestamp
     41    """
     42    return to_timestamp(now())
     43
     44
     45def now_ds() -> str:
     46    """
     47    Current utc ds.
     48
     49    Returns:
     50        Today's ds string.
     51    """
     52    return to_ds(now())
     53
     54
     55def yesterday() -> datetime:
     56    """
     57    Yesterday utc datetime.
     58
     59    Returns:
     60        A datetime object with tz utc representing yesterday's date
     61    """
     62    return to_datetime("yesterday")
     63
     64
     65def yesterday_ds() -> str:
     66    """
     67    Yesterday utc ds.
     68
     69    Returns:
     70        Yesterday's ds string.
     71    """
     72    return to_ds("yesterday")
     73
     74
     75def yesterday_timestamp() -> int:
     76    """
     77    Yesterday utc timestamp.
     78
     79    Returns:
     80        UTC epoch millis timestamp of yesterday
     81    """
     82    return to_timestamp(yesterday())
     83
     84
     85def to_timestamp(value: TimeLike, relative_base: t.Optional[datetime] = None) -> int:
     86    """
     87    Converts a value into an epoch millis timestamp.
     88
     89    Args:
     90        value: A variety of date formats. If value is a string, it must be in iso format.
     91        relative_base: The datetime to reference for time expressions that are using relative terms
     92
     93    Returns:
     94        Epoch millis timestamp.
     95    """
     96    return int(to_datetime(value, relative_base=relative_base).timestamp() * 1000)
     97
     98
     99def to_datetime(value: TimeLike, relative_base: t.Optional[datetime] = None) -> datetime:
    100    """Converts a value into a UTC datetime object.
    101
    102    Args:
    103        value: A variety of date formats. If the value is number-like, it is assumed to be millisecond epochs
    104        if it is larger than MILLIS_THRESHOLD.
    105        relative_base: The datetime to reference for time expressions that are using relative terms
    106
    107    Raises:
    108        ValueError if value cannot be converted to a datetime.
    109
    110    Returns:
    111        A datetime object with tz utc.
    112    """
    113    if isinstance(value, datetime):
    114        dt: t.Optional[datetime] = value
    115    elif isinstance(value, date):
    116        dt = datetime(value.year, value.month, value.day)
    117    elif isinstance(value, exp.Expression):
    118        return to_datetime(value.name)
    119    else:
    120        try:
    121            epoch = float(value)
    122        except ValueError:
    123            epoch = None
    124
    125        if epoch is None:
    126            dt = dateparser.parse(str(value), settings={"RELATIVE_BASE": relative_base or now()})
    127        else:
    128            try:
    129                dt = datetime.strptime(str(value), DATE_INT_FMT)
    130            except ValueError:
    131                dt = datetime.fromtimestamp(
    132                    epoch / 1000.0 if epoch > MILLIS_THRESHOLD else epoch, tz=UTC
    133                )
    134
    135    if dt is None:
    136        raise ValueError(f"Could not convert `{value}` to datetime.")
    137
    138    if dt.tzinfo:
    139        return dt.astimezone(UTC)
    140    return dt.replace(tzinfo=UTC)
    141
    142
    143def to_date(value: TimeLike, relative_base: t.Optional[datetime] = None) -> date:
    144    """Converts a value into a UTC date object
    145    Args:
    146        value: A variety of date formats. If the value is number-like, it is assumed to be millisecond epochs
    147        if it is larger than MILLIS_THRESHOLD.
    148        relative_base: The datetime to reference for time expressions that are using relative terms
    149
    150    Raises:
    151        ValueError if value cannot be converted to a date.
    152
    153    Returns:
    154        A date object with tz utc.
    155    """
    156    return to_datetime(value, relative_base).date()
    157
    158
    159def date_dict(
    160    start: TimeLike, end: TimeLike, latest: TimeLike, only_latest: bool = False
    161) -> t.Dict[str, t.Union[str, datetime, float, int]]:
    162    """Creates a kwarg dictionary of datetime variables for use in SQL Contexts.
    163
    164    Keys are like start_date, start_ds, end_date, end_ds...
    165
    166    Args:
    167        start: Start time.
    168        end: End time.
    169        latest: Latest time.
    170        only_latest: Only the latest timestamps will be returned.
    171
    172    Returns:
    173        A dictionary with various keys pointing to datetime formats.
    174    """
    175    kwargs: t.Dict[str, t.Union[str, datetime, float, int]] = {}
    176
    177    prefixes = [("latest", to_datetime(latest))]
    178
    179    if not only_latest:
    180        prefixes.append(("start", to_datetime(start)))
    181        prefixes.append(("end", to_datetime(end)))
    182
    183    for prefix, time_like in prefixes:
    184        dt = to_datetime(time_like)
    185        millis = to_timestamp(time_like)
    186        kwargs[f"{prefix}_date"] = dt
    187        kwargs[f"{prefix}_ds"] = to_ds(time_like)
    188        kwargs[f"{prefix}_ts"] = dt.isoformat()
    189        kwargs[f"{prefix}_epoch"] = millis / 1000
    190        kwargs[f"{prefix}_millis"] = millis
    191    return kwargs
    192
    193
    194def to_ds(obj: TimeLike) -> str:
    195    """Converts a TimeLike object into YYYY-MM-DD formatted string."""
    196    return to_datetime(obj).isoformat()[0:10]
    197
    198
    199def is_date(obj: TimeLike) -> bool:
    200    """Checks if a TimeLike object should be treated like a date."""
    201    if isinstance(obj, date) and not isinstance(obj, datetime):
    202        return True
    203
    204    try:
    205        time.strptime(str(obj).replace("-", ""), DATE_INT_FMT)
    206        return True
    207    except ValueError:
    208        return False
    209
    210
    211def make_inclusive(start: TimeLike, end: TimeLike) -> Interval:
    212    """Adjust start and end times to to become inclusive datetimes.
    213
    214    SQLMesh treats start and end times as inclusive so that filters can be written as
    215
    216    SELECT * FROM x WHERE ds BETWEEN @start_ds AND @end_ds.
    217    SELECT * FROM x WHERE ts BETWEEN @start_ts AND @end_ts.
    218
    219    In the ds ('2020-01-01') case, because start_ds and end_ds are categorical, between works even if
    220    start_ds and end_ds are equivalent. However, when we move to ts ('2022-01-01 12:00:00'), because timestamps
    221    are numeric, using simple equality doesn't make sense. When the end is not a categorical date, then it is
    222    treated as an exclusive range and converted to inclusive by subtracting 1 millisecond.
    223
    224    Args:
    225        start: Start timelike object.
    226        end: End timelike object.
    227
    228    Example:
    229        >>> make_inclusive("2020-01-01", "2020-01-01")
    230        (datetime.datetime(2020, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), datetime.datetime(2020, 1, 1, 23, 59, 59, 999000, tzinfo=datetime.timezone.utc))
    231
    232    Returns:
    233        A tuple of inclusive datetime objects.
    234    """
    235    start_dt = to_datetime(start)
    236    end_dt = to_datetime(end)
    237    if is_date(end):
    238        end_dt = end_dt + timedelta(days=1)
    239    return (start_dt, end_dt - timedelta(milliseconds=1))
    240
    241
    242def preserve_time_like_kind(input_value: TimeLike, output_value: TimeLike) -> TimeLike:
    243    if is_date(input_value):
    244        return to_date(output_value)
    245    return output_value
    246
    247
    248def validate_date_range(
    249    start: t.Optional[TimeLike],
    250    end: t.Optional[TimeLike],
    251) -> None:
    252    if start and end and to_datetime(start) > to_datetime(end):
    253        raise ValueError(
    254            f"Start date / time ({start}) can't be greater than end date / time ({end})"
    255        )

def now() -> datetime.datetime: View Source

    26def now() -> datetime:
    27    """
    28    Current utc datetime.
    29
    30    Returns:
    31        A datetime object with tz utc.
    32    """
    33    return datetime.utcnow().replace(tzinfo=UTC)

Current utc datetime.

###### Returns:

> A datetime object with tz utc.

def now_timestamp() -> int: View Source

    36def now_timestamp() -> int:
    37    """
    38    Current utc timestamp.
    39
    40    Returns:
    41        UTC epoch millis timestamp
    42    """
    43    return to_timestamp(now())

Current utc timestamp.

###### Returns:

> UTC epoch millis timestamp

def now_ds() -> str: View Source

    46def now_ds() -> str:
    47    """
    48    Current utc ds.
    49
    50    Returns:
    51        Today's ds string.
    52    """
    53    return to_ds(now())

Current utc ds.

###### Returns:

> Today's ds string.

def yesterday() -> datetime.datetime: View Source

    56def yesterday() -> datetime:
    57    """
    58    Yesterday utc datetime.
    59
    60    Returns:
    61        A datetime object with tz utc representing yesterday's date
    62    """
    63    return to_datetime("yesterday")

Yesterday utc datetime.

###### Returns:

> A datetime object with tz utc representing yesterday's date

def yesterday_ds() -> str: View Source

    66def yesterday_ds() -> str:
    67    """
    68    Yesterday utc ds.
    69
    70    Returns:
    71        Yesterday's ds string.
    72    """
    73    return to_ds("yesterday")

Yesterday utc ds.

###### Returns:

> Yesterday's ds string.

def yesterday_timestamp() -> int: View Source

    76def yesterday_timestamp() -> int:
    77    """
    78    Yesterday utc timestamp.
    79
    80    Returns:
    81        UTC epoch millis timestamp of yesterday
    82    """
    83    return to_timestamp(yesterday())

Yesterday utc timestamp.

###### Returns:

> UTC epoch millis timestamp of yesterday

def to_timestamp( value: Union[datetime.date, datetime.datetime, str, int, float], relative_base: Optional[datetime.datetime] = None) -> int: View Source

    86def to_timestamp(value: TimeLike, relative_base: t.Optional[datetime] = None) -> int:
    87    """
    88    Converts a value into an epoch millis timestamp.
    89
    90    Args:
    91        value: A variety of date formats. If value is a string, it must be in iso format.
    92        relative_base: The datetime to reference for time expressions that are using relative terms
    93
    94    Returns:
    95        Epoch millis timestamp.
    96    """
    97    return int(to_datetime(value, relative_base=relative_base).timestamp() * 1000)

Converts a value into an epoch millis timestamp.

###### Arguments:

- value: A variety of date formats. If value is a string, it must be in iso format.
- relative_base: The datetime to reference for time expressions that are using relative terms

###### Returns:

> Epoch millis timestamp.

def to_datetime( value: Union[datetime.date, datetime.datetime, str, int, float], relative_base: Optional[datetime.datetime] = None) -> datetime.datetime: View Source

    100def to_datetime(value: TimeLike, relative_base: t.Optional[datetime] = None) -> datetime:
    101    """Converts a value into a UTC datetime object.
    102
    103    Args:
    104        value: A variety of date formats. If the value is number-like, it is assumed to be millisecond epochs
    105        if it is larger than MILLIS_THRESHOLD.
    106        relative_base: The datetime to reference for time expressions that are using relative terms
    107
    108    Raises:
    109        ValueError if value cannot be converted to a datetime.
    110
    111    Returns:
    112        A datetime object with tz utc.
    113    """
    114    if isinstance(value, datetime):
    115        dt: t.Optional[datetime] = value
    116    elif isinstance(value, date):
    117        dt = datetime(value.year, value.month, value.day)
    118    elif isinstance(value, exp.Expression):
    119        return to_datetime(value.name)
    120    else:
    121        try:
    122            epoch = float(value)
    123        except ValueError:
    124            epoch = None
    125
    126        if epoch is None:
    127            dt = dateparser.parse(str(value), settings={"RELATIVE_BASE": relative_base or now()})
    128        else:
    129            try:
    130                dt = datetime.strptime(str(value), DATE_INT_FMT)
    131            except ValueError:
    132                dt = datetime.fromtimestamp(
    133                    epoch / 1000.0 if epoch > MILLIS_THRESHOLD else epoch, tz=UTC
    134                )
    135
    136    if dt is None:
    137        raise ValueError(f"Could not convert `{value}` to datetime.")
    138
    139    if dt.tzinfo:
    140        return dt.astimezone(UTC)
    141    return dt.replace(tzinfo=UTC)

Converts a value into a UTC datetime object.

###### Arguments:

- value: A variety of date formats. If the value is number-like, it is assumed to be millisecond epochs
- if it is larger than MILLIS_THRESHOLD.
- relative_base: The datetime to reference for time expressions that are using relative terms

###### Raises:

- ValueError if value cannot be converted to a datetime.

###### Returns:

> A datetime object with tz utc.

def to_date( value: Union[datetime.date, datetime.datetime, str, int, float], relative_base: Optional[datetime.datetime] = None) -> datetime.date: View Source

    144def to_date(value: TimeLike, relative_base: t.Optional[datetime] = None) -> date:
    145    """Converts a value into a UTC date object
    146    Args:
    147        value: A variety of date formats. If the value is number-like, it is assumed to be millisecond epochs
    148        if it is larger than MILLIS_THRESHOLD.
    149        relative_base: The datetime to reference for time expressions that are using relative terms
    150
    151    Raises:
    152        ValueError if value cannot be converted to a date.
    153
    154    Returns:
    155        A date object with tz utc.
    156    """
    157    return to_datetime(value, relative_base).date()

Converts a value into a UTC date object

###### Arguments:

- value: A variety of date formats. If the value is number-like, it is assumed to be millisecond epochs
- if it is larger than MILLIS_THRESHOLD.
- relative_base: The datetime to reference for time expressions that are using relative terms

###### Raises:

- ValueError if value cannot be converted to a date.

###### Returns:

> A date object with tz utc.

def date_dict( start: Union[datetime.date, datetime.datetime, str, int, float], end: Union[datetime.date, datetime.datetime, str, int, float], latest: Union[datetime.date, datetime.datetime, str, int, float], only_latest: bool = False) -> Dict[str, Union[str, datetime.datetime, float, int]]: View Source

    160def date_dict(
    161    start: TimeLike, end: TimeLike, latest: TimeLike, only_latest: bool = False
    162) -> t.Dict[str, t.Union[str, datetime, float, int]]:
    163    """Creates a kwarg dictionary of datetime variables for use in SQL Contexts.
    164
    165    Keys are like start_date, start_ds, end_date, end_ds...
    166
    167    Args:
    168        start: Start time.
    169        end: End time.
    170        latest: Latest time.
    171        only_latest: Only the latest timestamps will be returned.
    172
    173    Returns:
    174        A dictionary with various keys pointing to datetime formats.
    175    """
    176    kwargs: t.Dict[str, t.Union[str, datetime, float, int]] = {}
    177
    178    prefixes = [("latest", to_datetime(latest))]
    179
    180    if not only_latest:
    181        prefixes.append(("start", to_datetime(start)))
    182        prefixes.append(("end", to_datetime(end)))
    183
    184    for prefix, time_like in prefixes:
    185        dt = to_datetime(time_like)
    186        millis = to_timestamp(time_like)
    187        kwargs[f"{prefix}_date"] = dt
    188        kwargs[f"{prefix}_ds"] = to_ds(time_like)
    189        kwargs[f"{prefix}_ts"] = dt.isoformat()
    190        kwargs[f"{prefix}_epoch"] = millis / 1000
    191        kwargs[f"{prefix}_millis"] = millis
    192    return kwargs

Creates a kwarg dictionary of datetime variables for use in SQL Contexts.

Keys are like start_date, start_ds, end_date, end_ds...

###### Arguments:

- start: Start time.
- end: End time.
- latest: Latest time.
- only_latest: Only the latest timestamps will be returned.

###### Returns:

> A dictionary with various keys pointing to datetime formats.

def to_ds(obj: Union[datetime.date, datetime.datetime, str, int, float]) -> str: View Source

    195def to_ds(obj: TimeLike) -> str:
    196    """Converts a TimeLike object into YYYY-MM-DD formatted string."""
    197    return to_datetime(obj).isoformat()[0:10]

Converts a TimeLike object into YYYY-MM-DD formatted string.

def is_date(obj: Union[datetime.date, datetime.datetime, str, int, float]) -> bool: View Source

    200def is_date(obj: TimeLike) -> bool:
    201    """Checks if a TimeLike object should be treated like a date."""
    202    if isinstance(obj, date) and not isinstance(obj, datetime):
    203        return True
    204
    205    try:
    206        time.strptime(str(obj).replace("-", ""), DATE_INT_FMT)
    207        return True
    208    except ValueError:
    209        return False

Checks if a TimeLike object should be treated like a date.

def make_inclusive( start: Union[datetime.date, datetime.datetime, str, int, float], end: Union[datetime.date, datetime.datetime, str, int, float]) -> Tuple[datetime.datetime, datetime.datetime]: View Source

    212def make_inclusive(start: TimeLike, end: TimeLike) -> Interval:
    213    """Adjust start and end times to to become inclusive datetimes.
    214
    215    SQLMesh treats start and end times as inclusive so that filters can be written as
    216
    217    SELECT * FROM x WHERE ds BETWEEN @start_ds AND @end_ds.
    218    SELECT * FROM x WHERE ts BETWEEN @start_ts AND @end_ts.
    219
    220    In the ds ('2020-01-01') case, because start_ds and end_ds are categorical, between works even if
    221    start_ds and end_ds are equivalent. However, when we move to ts ('2022-01-01 12:00:00'), because timestamps
    222    are numeric, using simple equality doesn't make sense. When the end is not a categorical date, then it is
    223    treated as an exclusive range and converted to inclusive by subtracting 1 millisecond.
    224
    225    Args:
    226        start: Start timelike object.
    227        end: End timelike object.
    228
    229    Example:
    230        >>> make_inclusive("2020-01-01", "2020-01-01")
    231        (datetime.datetime(2020, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), datetime.datetime(2020, 1, 1, 23, 59, 59, 999000, tzinfo=datetime.timezone.utc))
    232
    233    Returns:
    234        A tuple of inclusive datetime objects.
    235    """
    236    start_dt = to_datetime(start)
    237    end_dt = to_datetime(end)
    238    if is_date(end):
    239        end_dt = end_dt + timedelta(days=1)
    240    return (start_dt, end_dt - timedelta(milliseconds=1))

Adjust start and end times to to become inclusive datetimes.

SQLMesh treats start and end times as inclusive so that filters can be written as

SELECT * FROM x WHERE ds BETWEEN @start_ds AND @end_ds. SELECT * FROM x WHERE ts BETWEEN @start_ts AND @end_ts.

In the ds ('2020-01-01') case, because start_ds and end_ds are categorical, between works even if start_ds and end_ds are equivalent. However, when we move to ts ('2022-01-01 12:00:00'), because timestamps are numeric, using simple equality doesn't make sense. When the end is not a categorical date, then it is treated as an exclusive range and converted to inclusive by subtracting 1 millisecond.

###### Arguments:

- start: Start timelike object.
- end: End timelike object.

###### Example:

>     >>> make_inclusive("2020-01-01", "2020-01-01")
>     (datetime.datetime(2020, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), datetime.datetime(2020, 1, 1, 23, 59, 59, 999000, tzinfo=datetime.timezone.utc))

###### Returns:

> A tuple of inclusive datetime objects.

def preserve_time_like_kind( input_value: Union[datetime.date, datetime.datetime, str, int, float], output_value: Union[datetime.date, datetime.datetime, str, int, float]) -> Union[datetime.date, datetime.datetime, str, int, float]: View Source

    243def preserve_time_like_kind(input_value: TimeLike, output_value: TimeLike) -> TimeLike:
    244    if is_date(input_value):
    245        return to_date(output_value)
    246    return output_value

def validate_date_range( start: Union[datetime.date, datetime.datetime, str, int, float, NoneType], end: Union[datetime.date, datetime.datetime, str, int, float, NoneType]) -> None: View Source

    249def validate_date_range(
    250    start: t.Optional[TimeLike],
    251    end: t.Optional[TimeLike],
    252) -> None:
    253    if start and end and to_datetime(start) > to_datetime(end):
    254        raise ValueError(
    255            f"Start date / time ({start}) can't be greater than end date / time ({end})"
    256        )