sqlmesh.core.state_sync.base API documentation

Edit on GitHub

# sqlmesh.core.state_sync.base

View Source

      1from __future__ import annotations
      2
      3import abc
      4import typing as t
      5
      6from sqlmesh.core import scheduler
      7from sqlmesh.core.environment import Environment
      8from sqlmesh.core.snapshot import (
      9    Intervals,
     10    Snapshot,
     11    SnapshotId,
     12    SnapshotIdLike,
     13    SnapshotInfoLike,
     14    SnapshotNameVersionLike,
     15    SnapshotTableInfo,
     16)
     17from sqlmesh.utils.date import TimeLike, now, to_datetime
     18from sqlmesh.utils.errors import SQLMeshError
     19
     20
     21class StateReader(abc.ABC):
     22    """Abstract base class for read-only operations on snapshot and environment state."""
     23
     24    @abc.abstractmethod
     25    def get_snapshots(
     26        self, snapshot_ids: t.Optional[t.Iterable[SnapshotIdLike]]
     27    ) -> t.Dict[SnapshotId, Snapshot]:
     28        """Bulk fetch snapshots given the corresponding snapshot ids.
     29
     30        Args:
     31            snapshot_ids: Iterable of snapshot ids to get. If not provided all
     32                available snapshots will be returned.
     33
     34        Returns:
     35            A dictionary of snapshot ids to snapshots for ones that could be found.
     36        """
     37
     38    @abc.abstractmethod
     39    def get_snapshots_with_same_version(
     40        self, snapshots: t.Iterable[SnapshotNameVersionLike]
     41    ) -> t.List[Snapshot]:
     42        """Fetches all snapshots that share the same version as the snapshots.
     43
     44        The output includes the snapshots with the specified version.
     45
     46        Args:
     47            snapshots: The collection of target name / version pairs.
     48
     49        Returns:
     50            The list of Snapshot objects.
     51        """
     52
     53    @abc.abstractmethod
     54    def snapshots_exist(self, snapshot_ids: t.Iterable[SnapshotIdLike]) -> t.Set[SnapshotId]:
     55        """Checks if multiple snapshots exist in the state sync.
     56
     57        Args:
     58            snapshot_ids: Iterable of snapshot ids to bulk check.
     59
     60        Returns:
     61            A set of all the existing snapshot ids.
     62        """
     63
     64    @abc.abstractmethod
     65    def get_environment(self, environment: str) -> t.Optional[Environment]:
     66        """Fetches the environment if it exists.
     67
     68        Args:
     69            environment: The environment
     70
     71        Returns:
     72            The environment object.
     73        """
     74
     75    @abc.abstractmethod
     76    def get_environments(self) -> t.List[Environment]:
     77        """Fetches all environments.
     78
     79        Returns:
     80            A list of all environments.
     81        """
     82
     83    @abc.abstractmethod
     84    def get_snapshots_by_models(self, *names: str) -> t.List[Snapshot]:
     85        """Get all snapshots by model name.
     86
     87        Returns:
     88            The list of snapshots.
     89        """
     90
     91    def missing_intervals(
     92        self,
     93        env_or_snapshots: str | Environment | t.Iterable[Snapshot],
     94        start: t.Optional[TimeLike] = None,
     95        end: t.Optional[TimeLike] = None,
     96        latest: t.Optional[TimeLike] = None,
     97        restatements: t.Optional[t.Iterable[str]] = None,
     98    ) -> t.Dict[Snapshot, Intervals]:
     99        """Find missing intervals for an environment or a list of snapshots.
    100
    101        Args:
    102            env_or_snapshots: The environment or snapshots to find missing intervals for.
    103            start: The start of the time range to look for.
    104            end: The end of the time range to look for.
    105            latest: The latest datetime to use for non-incremental queries.
    106
    107        Returns:
    108            A dictionary of SnapshotId to Intervals.
    109        """
    110
    111        if isinstance(env_or_snapshots, str):
    112            env = self.get_environment(env_or_snapshots)
    113        elif isinstance(env_or_snapshots, Environment):
    114            env = env_or_snapshots
    115        else:
    116            env = None
    117
    118        if env:
    119            snapshots_by_id = self.get_snapshots(env.snapshots)
    120            start = start or env.start_at
    121            end = end or env.end_at
    122        elif isinstance(env_or_snapshots, str):
    123            snapshots_by_id = {}
    124        elif not isinstance(env_or_snapshots, Environment):
    125            snapshots_by_id = {snapshot.snapshot_id: snapshot for snapshot in env_or_snapshots}
    126        else:
    127            raise SQLMeshError("This shouldn't be possible.")
    128
    129        if not snapshots_by_id:
    130            return {}
    131
    132        unversioned = [snapshot for snapshot in snapshots_by_id.values() if not snapshot.version]
    133
    134        snapshots_by_id = {
    135            **snapshots_by_id,
    136            **(self.get_snapshots(unversioned) if unversioned else {}),
    137        }
    138
    139        all_snapshots = {
    140            **snapshots_by_id,
    141            **{
    142                snapshot.snapshot_id: snapshot
    143                for snapshot in self.get_snapshots_with_same_version(
    144                    snapshot for snapshot in snapshots_by_id.values() if snapshot.version
    145                )
    146            },
    147        }
    148
    149        missing = {}
    150        start_date = to_datetime(start or scheduler.earliest_start_date(snapshots_by_id.values()))
    151        end_date = end or now()
    152        restatements = set(restatements or [])
    153
    154        for snapshot in Snapshot.merge_snapshots(snapshots_by_id, all_snapshots):
    155            if snapshot.name in restatements:
    156                snapshot.remove_interval(start_date, end_date)
    157            intervals = snapshot.missing_intervals(
    158                max(
    159                    start_date,
    160                    to_datetime(
    161                        scheduler.start_date(snapshot, snapshots_by_id.values()) or start_date
    162                    ),
    163                ),
    164                end_date,
    165                latest=latest,
    166            )
    167            if intervals:
    168                missing[snapshot] = intervals
    169        return missing
    170
    171
    172class StateSync(StateReader, abc.ABC):
    173    """Abstract base class for snapshot and environment state management."""
    174
    175    @abc.abstractmethod
    176    def init_schema(self) -> None:
    177        """Optional initialization of the sync."""
    178
    179    @abc.abstractmethod
    180    def push_snapshots(self, snapshots: t.Iterable[Snapshot]) -> None:
    181        """Push snapshots into the state sync.
    182
    183        This method only allows for pushing new snapshots. If existing snapshots are found,
    184        this method should raise an error.
    185
    186        Raises:
    187            SQLMeshError when existing snapshots are pushed.
    188
    189        Args:
    190            snapshots: A list of snapshots to save in the state sync.
    191        """
    192
    193    @abc.abstractmethod
    194    def delete_snapshots(self, snapshot_ids: t.Iterable[SnapshotIdLike]) -> None:
    195        """Delete snapshots from the state sync.
    196
    197        Args:
    198            snapshot_ids: A list of snapshot like objects to delete.
    199        """
    200
    201    @abc.abstractmethod
    202    def delete_expired_snapshots(self) -> t.List[Snapshot]:
    203        """Removes expired snapshots.
    204
    205        Expired snapshots are snapshots that have exceeded their time-to-live
    206        and are no longer in use within an environment.
    207
    208        Returns:
    209            The list of removed snapshots.
    210        """
    211
    212    @abc.abstractmethod
    213    def add_interval(
    214        self,
    215        snapshot_id: SnapshotIdLike,
    216        start: TimeLike,
    217        end: TimeLike,
    218        is_dev: bool = False,
    219    ) -> None:
    220        """Add an interval to a snapshot and sync it to the store.
    221
    222        Snapshots must be pushed before adding intervals to them.
    223
    224        Args:
    225            snapshot_id: The snapshot like object to add an interval to.
    226            start: The start of the interval to add.
    227            end: The end of the interval to add.
    228            is_dev: Indicates whether the given interval is being added while in
    229                development mode.
    230        """
    231
    232    @abc.abstractmethod
    233    def remove_interval(
    234        self,
    235        snapshots: t.Iterable[SnapshotInfoLike],
    236        start: TimeLike,
    237        end: TimeLike,
    238        all_snapshots: t.Optional[t.Iterable[Snapshot]] = None,
    239    ) -> None:
    240        """Remove an interval from a list of snapshots and sync it to the store.
    241
    242        Because multiple snapshots can be pointing to the same version or physical table, this method
    243        can also grab all snapshots tied to the passed in version.
    244
    245        Args:
    246            snapshots: The snapshot info like object to remove intervals from.
    247            start: The start of the interval to add.
    248            end: The end of the interval to add.
    249            all_snapshots: All snapshots can be passed in to skip fetching matching snapshot versions.
    250        """
    251
    252    @abc.abstractmethod
    253    def promote(
    254        self, environment: Environment, no_gaps: bool = False
    255    ) -> t.Tuple[t.List[SnapshotTableInfo], t.List[SnapshotTableInfo]]:
    256        """Update the environment to reflect the current state.
    257
    258        This method verifies that snapshots have been pushed.
    259
    260        Args:
    261            environment: The environment to promote.
    262            no_gaps:  Whether to ensure that new snapshots for models that are already a
    263                part of the target environment have no data gaps when compared against previous
    264                snapshots for same models.
    265
    266        Returns:
    267           A tuple of (added snapshot table infos, removed snapshot table infos)
    268        """
    269
    270    @abc.abstractmethod
    271    def delete_expired_environments(self) -> t.List[Environment]:
    272        """Removes expired environments.
    273
    274        Expired environments are environments that have exceeded their time-to-live value.
    275
    276        Returns:
    277            The list of removed environments.
    278        """
    279
    280    @abc.abstractmethod
    281    def unpause_snapshots(
    282        self, snapshots: t.Iterable[SnapshotInfoLike], unpaused_dt: TimeLike
    283    ) -> None:
    284        """Unpauses target snapshots.
    285
    286        Unpaused snapshots are scheduled for evaluation on a recurring basis.
    287        Once unpaused a snapshot can't be paused again.
    288
    289        Args:
    290            snapshots: Target snapshots.
    291            unpaused_dt: The datetime object which indicates when target snapshots
    292                were unpaused.
    293        """

class StateReader(abc.ABC): View Source

     22class StateReader(abc.ABC):
     23    """Abstract base class for read-only operations on snapshot and environment state."""
     24
     25    @abc.abstractmethod
     26    def get_snapshots(
     27        self, snapshot_ids: t.Optional[t.Iterable[SnapshotIdLike]]
     28    ) -> t.Dict[SnapshotId, Snapshot]:
     29        """Bulk fetch snapshots given the corresponding snapshot ids.
     30
     31        Args:
     32            snapshot_ids: Iterable of snapshot ids to get. If not provided all
     33                available snapshots will be returned.
     34
     35        Returns:
     36            A dictionary of snapshot ids to snapshots for ones that could be found.
     37        """
     38
     39    @abc.abstractmethod
     40    def get_snapshots_with_same_version(
     41        self, snapshots: t.Iterable[SnapshotNameVersionLike]
     42    ) -> t.List[Snapshot]:
     43        """Fetches all snapshots that share the same version as the snapshots.
     44
     45        The output includes the snapshots with the specified version.
     46
     47        Args:
     48            snapshots: The collection of target name / version pairs.
     49
     50        Returns:
     51            The list of Snapshot objects.
     52        """
     53
     54    @abc.abstractmethod
     55    def snapshots_exist(self, snapshot_ids: t.Iterable[SnapshotIdLike]) -> t.Set[SnapshotId]:
     56        """Checks if multiple snapshots exist in the state sync.
     57
     58        Args:
     59            snapshot_ids: Iterable of snapshot ids to bulk check.
     60
     61        Returns:
     62            A set of all the existing snapshot ids.
     63        """
     64
     65    @abc.abstractmethod
     66    def get_environment(self, environment: str) -> t.Optional[Environment]:
     67        """Fetches the environment if it exists.
     68
     69        Args:
     70            environment: The environment
     71
     72        Returns:
     73            The environment object.
     74        """
     75
     76    @abc.abstractmethod
     77    def get_environments(self) -> t.List[Environment]:
     78        """Fetches all environments.
     79
     80        Returns:
     81            A list of all environments.
     82        """
     83
     84    @abc.abstractmethod
     85    def get_snapshots_by_models(self, *names: str) -> t.List[Snapshot]:
     86        """Get all snapshots by model name.
     87
     88        Returns:
     89            The list of snapshots.
     90        """
     91
     92    def missing_intervals(
     93        self,
     94        env_or_snapshots: str | Environment | t.Iterable[Snapshot],
     95        start: t.Optional[TimeLike] = None,
     96        end: t.Optional[TimeLike] = None,
     97        latest: t.Optional[TimeLike] = None,
     98        restatements: t.Optional[t.Iterable[str]] = None,
     99    ) -> t.Dict[Snapshot, Intervals]:
    100        """Find missing intervals for an environment or a list of snapshots.
    101
    102        Args:
    103            env_or_snapshots: The environment or snapshots to find missing intervals for.
    104            start: The start of the time range to look for.
    105            end: The end of the time range to look for.
    106            latest: The latest datetime to use for non-incremental queries.
    107
    108        Returns:
    109            A dictionary of SnapshotId to Intervals.
    110        """
    111
    112        if isinstance(env_or_snapshots, str):
    113            env = self.get_environment(env_or_snapshots)
    114        elif isinstance(env_or_snapshots, Environment):
    115            env = env_or_snapshots
    116        else:
    117            env = None
    118
    119        if env:
    120            snapshots_by_id = self.get_snapshots(env.snapshots)
    121            start = start or env.start_at
    122            end = end or env.end_at
    123        elif isinstance(env_or_snapshots, str):
    124            snapshots_by_id = {}
    125        elif not isinstance(env_or_snapshots, Environment):
    126            snapshots_by_id = {snapshot.snapshot_id: snapshot for snapshot in env_or_snapshots}
    127        else:
    128            raise SQLMeshError("This shouldn't be possible.")
    129
    130        if not snapshots_by_id:
    131            return {}
    132
    133        unversioned = [snapshot for snapshot in snapshots_by_id.values() if not snapshot.version]
    134
    135        snapshots_by_id = {
    136            **snapshots_by_id,
    137            **(self.get_snapshots(unversioned) if unversioned else {}),
    138        }
    139
    140        all_snapshots = {
    141            **snapshots_by_id,
    142            **{
    143                snapshot.snapshot_id: snapshot
    144                for snapshot in self.get_snapshots_with_same_version(
    145                    snapshot for snapshot in snapshots_by_id.values() if snapshot.version
    146                )
    147            },
    148        }
    149
    150        missing = {}
    151        start_date = to_datetime(start or scheduler.earliest_start_date(snapshots_by_id.values()))
    152        end_date = end or now()
    153        restatements = set(restatements or [])
    154
    155        for snapshot in Snapshot.merge_snapshots(snapshots_by_id, all_snapshots):
    156            if snapshot.name in restatements:
    157                snapshot.remove_interval(start_date, end_date)
    158            intervals = snapshot.missing_intervals(
    159                max(
    160                    start_date,
    161                    to_datetime(
    162                        scheduler.start_date(snapshot, snapshots_by_id.values()) or start_date
    163                    ),
    164                ),
    165                end_date,
    166                latest=latest,
    167            )
    168            if intervals:
    169                missing[snapshot] = intervals
    170        return missing

Abstract base class for read-only operations on snapshot and environment state.

@abc.abstractmethod
def get_snapshots( self, snapshot_ids: Optional[Iterable[Union[sqlmesh.core.snapshot.definition.SnapshotId, sqlmesh.core.snapshot.definition.SnapshotTableInfo, sqlmesh.core.snapshot.definition.Snapshot]]]) -> Dict[sqlmesh.core.snapshot.definition.SnapshotId, sqlmesh.core.snapshot.definition.Snapshot]: View Source

    25    @abc.abstractmethod
    26    def get_snapshots(
    27        self, snapshot_ids: t.Optional[t.Iterable[SnapshotIdLike]]
    28    ) -> t.Dict[SnapshotId, Snapshot]:
    29        """Bulk fetch snapshots given the corresponding snapshot ids.
    30
    31        Args:
    32            snapshot_ids: Iterable of snapshot ids to get. If not provided all
    33                available snapshots will be returned.
    34
    35        Returns:
    36            A dictionary of snapshot ids to snapshots for ones that could be found.
    37        """

Bulk fetch snapshots given the corresponding snapshot ids.

###### Arguments:

- snapshot_ids: Iterable of snapshot ids to get. If not provided all available snapshots will be returned.

###### Returns:

> A dictionary of snapshot ids to snapshots for ones that could be found.

@abc.abstractmethod
def get_snapshots_with_same_version( self, snapshots: Iterable[Union[sqlmesh.core.snapshot.definition.SnapshotNameVersion, sqlmesh.core.snapshot.definition.SnapshotTableInfo, sqlmesh.core.snapshot.definition.Snapshot]]) -> List[sqlmesh.core.snapshot.definition.Snapshot]: View Source

    39    @abc.abstractmethod
    40    def get_snapshots_with_same_version(
    41        self, snapshots: t.Iterable[SnapshotNameVersionLike]
    42    ) -> t.List[Snapshot]:
    43        """Fetches all snapshots that share the same version as the snapshots.
    44
    45        The output includes the snapshots with the specified version.
    46
    47        Args:
    48            snapshots: The collection of target name / version pairs.
    49
    50        Returns:
    51            The list of Snapshot objects.
    52        """

Fetches all snapshots that share the same version as the snapshots.

The output includes the snapshots with the specified version.

###### Arguments:

- snapshots: The collection of target name / version pairs.

###### Returns:

> The list of Snapshot objects.

@abc.abstractmethod
def snapshots_exist( self, snapshot_ids: Iterable[Union[sqlmesh.core.snapshot.definition.SnapshotId, sqlmesh.core.snapshot.definition.SnapshotTableInfo, sqlmesh.core.snapshot.definition.Snapshot]]) -> Set[sqlmesh.core.snapshot.definition.SnapshotId]: View Source

    54    @abc.abstractmethod
    55    def snapshots_exist(self, snapshot_ids: t.Iterable[SnapshotIdLike]) -> t.Set[SnapshotId]:
    56        """Checks if multiple snapshots exist in the state sync.
    57
    58        Args:
    59            snapshot_ids: Iterable of snapshot ids to bulk check.
    60
    61        Returns:
    62            A set of all the existing snapshot ids.
    63        """

Checks if multiple snapshots exist in the state sync.

###### Arguments:

- snapshot_ids: Iterable of snapshot ids to bulk check.

###### Returns:

> A set of all the existing snapshot ids.

@abc.abstractmethod
def get_environment(self, environment: str) -> Optional[sqlmesh.core.environment.Environment]: View Source

    65    @abc.abstractmethod
    66    def get_environment(self, environment: str) -> t.Optional[Environment]:
    67        """Fetches the environment if it exists.
    68
    69        Args:
    70            environment: The environment
    71
    72        Returns:
    73            The environment object.
    74        """

Fetches the environment if it exists.

###### Arguments:

- environment: The environment

###### Returns:

> The environment object.

@abc.abstractmethod
def get_environments(self) -> List[sqlmesh.core.environment.Environment]: View Source

    76    @abc.abstractmethod
    77    def get_environments(self) -> t.List[Environment]:
    78        """Fetches all environments.
    79
    80        Returns:
    81            A list of all environments.
    82        """

Fetches all environments.

###### Returns:

> A list of all environments.

@abc.abstractmethod
def get_snapshots_by_models(self, *names: str) -> List[sqlmesh.core.snapshot.definition.Snapshot]: View Source

    84    @abc.abstractmethod
    85    def get_snapshots_by_models(self, *names: str) -> t.List[Snapshot]:
    86        """Get all snapshots by model name.
    87
    88        Returns:
    89            The list of snapshots.
    90        """

Get all snapshots by model name.

###### Returns:

> The list of snapshots.

def missing_intervals( self, env_or_snapshots: Union[str, sqlmesh.core.environment.Environment, Iterable[sqlmesh.core.snapshot.definition.Snapshot]], start: Union[datetime.date, datetime.datetime, str, int, float, NoneType] = None, end: Union[datetime.date, datetime.datetime, str, int, float, NoneType] = None, latest: Union[datetime.date, datetime.datetime, str, int, float, NoneType] = None, restatements: Optional[Iterable[str]] = None) -> Dict[sqlmesh.core.snapshot.definition.Snapshot, List[Tuple[int, int]]]: View Source

     92    def missing_intervals(
     93        self,
     94        env_or_snapshots: str | Environment | t.Iterable[Snapshot],
     95        start: t.Optional[TimeLike] = None,
     96        end: t.Optional[TimeLike] = None,
     97        latest: t.Optional[TimeLike] = None,
     98        restatements: t.Optional[t.Iterable[str]] = None,
     99    ) -> t.Dict[Snapshot, Intervals]:
    100        """Find missing intervals for an environment or a list of snapshots.
    101
    102        Args:
    103            env_or_snapshots: The environment or snapshots to find missing intervals for.
    104            start: The start of the time range to look for.
    105            end: The end of the time range to look for.
    106            latest: The latest datetime to use for non-incremental queries.
    107
    108        Returns:
    109            A dictionary of SnapshotId to Intervals.
    110        """
    111
    112        if isinstance(env_or_snapshots, str):
    113            env = self.get_environment(env_or_snapshots)
    114        elif isinstance(env_or_snapshots, Environment):
    115            env = env_or_snapshots
    116        else:
    117            env = None
    118
    119        if env:
    120            snapshots_by_id = self.get_snapshots(env.snapshots)
    121            start = start or env.start_at
    122            end = end or env.end_at
    123        elif isinstance(env_or_snapshots, str):
    124            snapshots_by_id = {}
    125        elif not isinstance(env_or_snapshots, Environment):
    126            snapshots_by_id = {snapshot.snapshot_id: snapshot for snapshot in env_or_snapshots}
    127        else:
    128            raise SQLMeshError("This shouldn't be possible.")
    129
    130        if not snapshots_by_id:
    131            return {}
    132
    133        unversioned = [snapshot for snapshot in snapshots_by_id.values() if not snapshot.version]
    134
    135        snapshots_by_id = {
    136            **snapshots_by_id,
    137            **(self.get_snapshots(unversioned) if unversioned else {}),
    138        }
    139
    140        all_snapshots = {
    141            **snapshots_by_id,
    142            **{
    143                snapshot.snapshot_id: snapshot
    144                for snapshot in self.get_snapshots_with_same_version(
    145                    snapshot for snapshot in snapshots_by_id.values() if snapshot.version
    146                )
    147            },
    148        }
    149
    150        missing = {}
    151        start_date = to_datetime(start or scheduler.earliest_start_date(snapshots_by_id.values()))
    152        end_date = end or now()
    153        restatements = set(restatements or [])
    154
    155        for snapshot in Snapshot.merge_snapshots(snapshots_by_id, all_snapshots):
    156            if snapshot.name in restatements:
    157                snapshot.remove_interval(start_date, end_date)
    158            intervals = snapshot.missing_intervals(
    159                max(
    160                    start_date,
    161                    to_datetime(
    162                        scheduler.start_date(snapshot, snapshots_by_id.values()) or start_date
    163                    ),
    164                ),
    165                end_date,
    166                latest=latest,
    167            )
    168            if intervals:
    169                missing[snapshot] = intervals
    170        return missing

Find missing intervals for an environment or a list of snapshots.

###### Arguments:

- env_or_snapshots: The environment or snapshots to find missing intervals for.
- start: The start of the time range to look for.
- end: The end of the time range to look for.
- latest: The latest datetime to use for non-incremental queries.

###### Returns:

> A dictionary of SnapshotId to Intervals.

class StateSync(StateReader, abc.ABC): View Source

    173class StateSync(StateReader, abc.ABC):
    174    """Abstract base class for snapshot and environment state management."""
    175
    176    @abc.abstractmethod
    177    def init_schema(self) -> None:
    178        """Optional initialization of the sync."""
    179
    180    @abc.abstractmethod
    181    def push_snapshots(self, snapshots: t.Iterable[Snapshot]) -> None:
    182        """Push snapshots into the state sync.
    183
    184        This method only allows for pushing new snapshots. If existing snapshots are found,
    185        this method should raise an error.
    186
    187        Raises:
    188            SQLMeshError when existing snapshots are pushed.
    189
    190        Args:
    191            snapshots: A list of snapshots to save in the state sync.
    192        """
    193
    194    @abc.abstractmethod
    195    def delete_snapshots(self, snapshot_ids: t.Iterable[SnapshotIdLike]) -> None:
    196        """Delete snapshots from the state sync.
    197
    198        Args:
    199            snapshot_ids: A list of snapshot like objects to delete.
    200        """
    201
    202    @abc.abstractmethod
    203    def delete_expired_snapshots(self) -> t.List[Snapshot]:
    204        """Removes expired snapshots.
    205
    206        Expired snapshots are snapshots that have exceeded their time-to-live
    207        and are no longer in use within an environment.
    208
    209        Returns:
    210            The list of removed snapshots.
    211        """
    212
    213    @abc.abstractmethod
    214    def add_interval(
    215        self,
    216        snapshot_id: SnapshotIdLike,
    217        start: TimeLike,
    218        end: TimeLike,
    219        is_dev: bool = False,
    220    ) -> None:
    221        """Add an interval to a snapshot and sync it to the store.
    222
    223        Snapshots must be pushed before adding intervals to them.
    224
    225        Args:
    226            snapshot_id: The snapshot like object to add an interval to.
    227            start: The start of the interval to add.
    228            end: The end of the interval to add.
    229            is_dev: Indicates whether the given interval is being added while in
    230                development mode.
    231        """
    232
    233    @abc.abstractmethod
    234    def remove_interval(
    235        self,
    236        snapshots: t.Iterable[SnapshotInfoLike],
    237        start: TimeLike,
    238        end: TimeLike,
    239        all_snapshots: t.Optional[t.Iterable[Snapshot]] = None,
    240    ) -> None:
    241        """Remove an interval from a list of snapshots and sync it to the store.
    242
    243        Because multiple snapshots can be pointing to the same version or physical table, this method
    244        can also grab all snapshots tied to the passed in version.
    245
    246        Args:
    247            snapshots: The snapshot info like object to remove intervals from.
    248            start: The start of the interval to add.
    249            end: The end of the interval to add.
    250            all_snapshots: All snapshots can be passed in to skip fetching matching snapshot versions.
    251        """
    252
    253    @abc.abstractmethod
    254    def promote(
    255        self, environment: Environment, no_gaps: bool = False
    256    ) -> t.Tuple[t.List[SnapshotTableInfo], t.List[SnapshotTableInfo]]:
    257        """Update the environment to reflect the current state.
    258
    259        This method verifies that snapshots have been pushed.
    260
    261        Args:
    262            environment: The environment to promote.
    263            no_gaps:  Whether to ensure that new snapshots for models that are already a
    264                part of the target environment have no data gaps when compared against previous
    265                snapshots for same models.
    266
    267        Returns:
    268           A tuple of (added snapshot table infos, removed snapshot table infos)
    269        """
    270
    271    @abc.abstractmethod
    272    def delete_expired_environments(self) -> t.List[Environment]:
    273        """Removes expired environments.
    274
    275        Expired environments are environments that have exceeded their time-to-live value.
    276
    277        Returns:
    278            The list of removed environments.
    279        """
    280
    281    @abc.abstractmethod
    282    def unpause_snapshots(
    283        self, snapshots: t.Iterable[SnapshotInfoLike], unpaused_dt: TimeLike
    284    ) -> None:
    285        """Unpauses target snapshots.
    286
    287        Unpaused snapshots are scheduled for evaluation on a recurring basis.
    288        Once unpaused a snapshot can't be paused again.
    289
    290        Args:
    291            snapshots: Target snapshots.
    292            unpaused_dt: The datetime object which indicates when target snapshots
    293                were unpaused.
    294        """

Abstract base class for snapshot and environment state management.

@abc.abstractmethod
def init_schema(self) -> None: View Source

    176    @abc.abstractmethod
    177    def init_schema(self) -> None:
    178        """Optional initialization of the sync."""

Optional initialization of the sync.

@abc.abstractmethod
def push_snapshots( self, snapshots: Iterable[sqlmesh.core.snapshot.definition.Snapshot]) -> None: View Source

    180    @abc.abstractmethod
    181    def push_snapshots(self, snapshots: t.Iterable[Snapshot]) -> None:
    182        """Push snapshots into the state sync.
    183
    184        This method only allows for pushing new snapshots. If existing snapshots are found,
    185        this method should raise an error.
    186
    187        Raises:
    188            SQLMeshError when existing snapshots are pushed.
    189
    190        Args:
    191            snapshots: A list of snapshots to save in the state sync.
    192        """

Push snapshots into the state sync.

This method only allows for pushing new snapshots. If existing snapshots are found, this method should raise an error.

###### Raises:

- SQLMeshError when existing snapshots are pushed.

###### Arguments:

- snapshots: A list of snapshots to save in the state sync.

@abc.abstractmethod
def delete_snapshots( self, snapshot_ids: Iterable[Union[sqlmesh.core.snapshot.definition.SnapshotId, sqlmesh.core.snapshot.definition.SnapshotTableInfo, sqlmesh.core.snapshot.definition.Snapshot]]) -> None: View Source

    194    @abc.abstractmethod
    195    def delete_snapshots(self, snapshot_ids: t.Iterable[SnapshotIdLike]) -> None:
    196        """Delete snapshots from the state sync.
    197
    198        Args:
    199            snapshot_ids: A list of snapshot like objects to delete.
    200        """

Delete snapshots from the state sync.

###### Arguments:

- snapshot_ids: A list of snapshot like objects to delete.

@abc.abstractmethod
def delete_expired_snapshots(self) -> List[sqlmesh.core.snapshot.definition.Snapshot]: View Source

    202    @abc.abstractmethod
    203    def delete_expired_snapshots(self) -> t.List[Snapshot]:
    204        """Removes expired snapshots.
    205
    206        Expired snapshots are snapshots that have exceeded their time-to-live
    207        and are no longer in use within an environment.
    208
    209        Returns:
    210            The list of removed snapshots.
    211        """

Removes expired snapshots.

Expired snapshots are snapshots that have exceeded their time-to-live and are no longer in use within an environment.

###### Returns:

> The list of removed snapshots.

@abc.abstractmethod
def add_interval( self, snapshot_id: Union[sqlmesh.core.snapshot.definition.SnapshotId, sqlmesh.core.snapshot.definition.SnapshotTableInfo, sqlmesh.core.snapshot.definition.Snapshot], start: Union[datetime.date, datetime.datetime, str, int, float], end: Union[datetime.date, datetime.datetime, str, int, float], is_dev: bool = False) -> None: View Source

    213    @abc.abstractmethod
    214    def add_interval(
    215        self,
    216        snapshot_id: SnapshotIdLike,
    217        start: TimeLike,
    218        end: TimeLike,
    219        is_dev: bool = False,
    220    ) -> None:
    221        """Add an interval to a snapshot and sync it to the store.
    222
    223        Snapshots must be pushed before adding intervals to them.
    224
    225        Args:
    226            snapshot_id: The snapshot like object to add an interval to.
    227            start: The start of the interval to add.
    228            end: The end of the interval to add.
    229            is_dev: Indicates whether the given interval is being added while in
    230                development mode.
    231        """

Add an interval to a snapshot and sync it to the store.

Snapshots must be pushed before adding intervals to them.

###### Arguments:

- snapshot_id: The snapshot like object to add an interval to.
- start: The start of the interval to add.
- end: The end of the interval to add.
- is_dev: Indicates whether the given interval is being added while in development mode.

@abc.abstractmethod
def remove_interval( self, snapshots: Iterable[Union[sqlmesh.core.snapshot.definition.SnapshotTableInfo, sqlmesh.core.snapshot.definition.Snapshot]], start: Union[datetime.date, datetime.datetime, str, int, float], end: Union[datetime.date, datetime.datetime, str, int, float], all_snapshots: Optional[Iterable[sqlmesh.core.snapshot.definition.Snapshot]] = None) -> None: View Source

    233    @abc.abstractmethod
    234    def remove_interval(
    235        self,
    236        snapshots: t.Iterable[SnapshotInfoLike],
    237        start: TimeLike,
    238        end: TimeLike,
    239        all_snapshots: t.Optional[t.Iterable[Snapshot]] = None,
    240    ) -> None:
    241        """Remove an interval from a list of snapshots and sync it to the store.
    242
    243        Because multiple snapshots can be pointing to the same version or physical table, this method
    244        can also grab all snapshots tied to the passed in version.
    245
    246        Args:
    247            snapshots: The snapshot info like object to remove intervals from.
    248            start: The start of the interval to add.
    249            end: The end of the interval to add.
    250            all_snapshots: All snapshots can be passed in to skip fetching matching snapshot versions.
    251        """

Remove an interval from a list of snapshots and sync it to the store.

Because multiple snapshots can be pointing to the same version or physical table, this method can also grab all snapshots tied to the passed in version.

###### Arguments:

- snapshots: The snapshot info like object to remove intervals from.
- start: The start of the interval to add.
- end: The end of the interval to add.
- all_snapshots: All snapshots can be passed in to skip fetching matching snapshot versions.

@abc.abstractmethod
def promote( self, environment: sqlmesh.core.environment.Environment, no_gaps: bool = False) -> Tuple[List[sqlmesh.core.snapshot.definition.SnapshotTableInfo], List[sqlmesh.core.snapshot.definition.SnapshotTableInfo]]: View Source

    253    @abc.abstractmethod
    254    def promote(
    255        self, environment: Environment, no_gaps: bool = False
    256    ) -> t.Tuple[t.List[SnapshotTableInfo], t.List[SnapshotTableInfo]]:
    257        """Update the environment to reflect the current state.
    258
    259        This method verifies that snapshots have been pushed.
    260
    261        Args:
    262            environment: The environment to promote.
    263            no_gaps:  Whether to ensure that new snapshots for models that are already a
    264                part of the target environment have no data gaps when compared against previous
    265                snapshots for same models.
    266
    267        Returns:
    268           A tuple of (added snapshot table infos, removed snapshot table infos)
    269        """

Update the environment to reflect the current state.

This method verifies that snapshots have been pushed.

###### Arguments:

- environment: The environment to promote.
- no_gaps: Whether to ensure that new snapshots for models that are already a part of the target environment have no data gaps when compared against previous snapshots for same models.

###### Returns:

> A tuple of (added snapshot table infos, removed snapshot table infos)

@abc.abstractmethod
def delete_expired_environments(self) -> List[sqlmesh.core.environment.Environment]: View Source

    271    @abc.abstractmethod
    272    def delete_expired_environments(self) -> t.List[Environment]:
    273        """Removes expired environments.
    274
    275        Expired environments are environments that have exceeded their time-to-live value.
    276
    277        Returns:
    278            The list of removed environments.
    279        """

Removes expired environments.

Expired environments are environments that have exceeded their time-to-live value.

###### Returns:

> The list of removed environments.

@abc.abstractmethod
def unpause_snapshots( self, snapshots: Iterable[Union[sqlmesh.core.snapshot.definition.SnapshotTableInfo, sqlmesh.core.snapshot.definition.Snapshot]], unpaused_dt: Union[datetime.date, datetime.datetime, str, int, float]) -> None: View Source

    281    @abc.abstractmethod
    282    def unpause_snapshots(
    283        self, snapshots: t.Iterable[SnapshotInfoLike], unpaused_dt: TimeLike
    284    ) -> None:
    285        """Unpauses target snapshots.
    286
    287        Unpaused snapshots are scheduled for evaluation on a recurring basis.
    288        Once unpaused a snapshot can't be paused again.
    289
    290        Args:
    291            snapshots: Target snapshots.
    292            unpaused_dt: The datetime object which indicates when target snapshots
    293                were unpaused.
    294        """

Unpauses target snapshots.

Unpaused snapshots are scheduled for evaluation on a recurring basis. Once unpaused a snapshot can't be paused again.

###### Arguments:

- snapshots: Target snapshots.
- unpaused_dt: The datetime object which indicates when target snapshots were unpaused.

##### Inherited Members

StateReader
  get_snapshots
  get_snapshots_with_same_version
  snapshots_exist
  get_environment
  get_environments
  get_snapshots_by_models
  missing_intervals