sqlmesh.engines.commands API documentation

Edit on GitHub

# sqlmesh.engines.commands

View Source

      1import typing as t
      2from enum import Enum
      3
      4from sqlmesh.core.environment import Environment
      5from sqlmesh.core.snapshot import (
      6    Snapshot,
      7    SnapshotEvaluator,
      8    SnapshotId,
      9    SnapshotTableInfo,
     10)
     11from sqlmesh.utils.date import TimeLike
     12from sqlmesh.utils.pydantic import PydanticModel
     13
     14COMMAND_PAYLOAD_FILE_NAME = "payload.json"
     15
     16
     17class CommandType(str, Enum):
     18    EVALUATE = "evaluate"
     19    PROMOTE = "promote"
     20    DEMOTE = "demote"
     21    CLEANUP = "cleanup"
     22    CREATE_TABLES = "create_tables"
     23    MIGRATE_TABLES = "migrate_tables"
     24
     25    # This makes it easy to integrate with argparse
     26    def __str__(self) -> str:
     27        return self.value
     28
     29
     30class EvaluateCommandPayload(PydanticModel):
     31    snapshot: Snapshot
     32    parent_snapshots: t.Dict[str, Snapshot]
     33    start: TimeLike
     34    end: TimeLike
     35    latest: TimeLike
     36    is_dev: bool
     37
     38
     39class PromoteCommandPayload(PydanticModel):
     40    snapshots: t.List[SnapshotTableInfo]
     41    environment: str
     42    is_dev: bool
     43
     44
     45class DemoteCommandPayload(PydanticModel):
     46    snapshots: t.List[SnapshotTableInfo]
     47    environment: str
     48
     49
     50class CleanupCommandPayload(PydanticModel):
     51    environments: t.List[Environment]
     52    snapshots: t.List[SnapshotTableInfo]
     53
     54
     55class CreateTablesCommandPayload(PydanticModel):
     56    target_snapshot_ids: t.List[SnapshotId]
     57    snapshots: t.List[Snapshot]
     58
     59
     60class MigrateTablesCommandPayload(PydanticModel):
     61    snapshots: t.List[SnapshotTableInfo]
     62
     63
     64def evaluate(
     65    evaluator: SnapshotEvaluator, command_payload: t.Union[str, EvaluateCommandPayload]
     66) -> None:
     67    if isinstance(command_payload, str):
     68        command_payload = EvaluateCommandPayload.parse_raw(command_payload)
     69
     70    parent_snapshots = command_payload.parent_snapshots
     71    parent_snapshots[command_payload.snapshot.name] = command_payload.snapshot
     72
     73    evaluator.evaluate(
     74        command_payload.snapshot,
     75        command_payload.start,
     76        command_payload.end,
     77        command_payload.latest,
     78        snapshots=parent_snapshots,
     79        is_dev=command_payload.is_dev,
     80    )
     81    evaluator.audit(
     82        snapshot=command_payload.snapshot,
     83        start=command_payload.start,
     84        end=command_payload.end,
     85        latest=command_payload.latest,
     86        snapshots=parent_snapshots,
     87        is_dev=command_payload.is_dev,
     88    )
     89
     90
     91def promote(
     92    evaluator: SnapshotEvaluator, command_payload: t.Union[str, PromoteCommandPayload]
     93) -> None:
     94    if isinstance(command_payload, str):
     95        command_payload = PromoteCommandPayload.parse_raw(command_payload)
     96    evaluator.promote(
     97        command_payload.snapshots,
     98        command_payload.environment,
     99        is_dev=command_payload.is_dev,
    100    )
    101
    102
    103def demote(
    104    evaluator: SnapshotEvaluator, command_payload: t.Union[str, DemoteCommandPayload]
    105) -> None:
    106    if isinstance(command_payload, str):
    107        command_payload = DemoteCommandPayload.parse_raw(command_payload)
    108    evaluator.demote(command_payload.snapshots, command_payload.environment)
    109
    110
    111def cleanup(
    112    evaluator: SnapshotEvaluator, command_payload: t.Union[str, CleanupCommandPayload]
    113) -> None:
    114    if isinstance(command_payload, str):
    115        command_payload = CleanupCommandPayload.parse_raw(command_payload)
    116    for environment in command_payload.environments:
    117        evaluator.demote(environment.snapshots, environment.name)
    118    evaluator.cleanup(command_payload.snapshots)
    119
    120
    121def create_tables(
    122    evaluator: SnapshotEvaluator,
    123    command_payload: t.Union[str, CreateTablesCommandPayload],
    124) -> None:
    125    if isinstance(command_payload, str):
    126        command_payload = CreateTablesCommandPayload.parse_raw(command_payload)
    127
    128    snapshots_by_id = {s.snapshot_id: s for s in command_payload.snapshots}
    129    target_snapshots = [snapshots_by_id[sid] for sid in command_payload.target_snapshot_ids]
    130    evaluator.create(target_snapshots, snapshots_by_id)
    131
    132
    133def migrate_tables(
    134    evaluator: SnapshotEvaluator,
    135    command_payload: t.Union[str, MigrateTablesCommandPayload],
    136) -> None:
    137    if isinstance(command_payload, str):
    138        command_payload = MigrateTablesCommandPayload.parse_raw(command_payload)
    139    evaluator.migrate(command_payload.snapshots)
    140
    141
    142COMMAND_HANDLERS: t.Dict[CommandType, t.Callable[[SnapshotEvaluator, str], None]] = {
    143    CommandType.EVALUATE: evaluate,
    144    CommandType.PROMOTE: promote,
    145    CommandType.DEMOTE: demote,
    146    CommandType.CLEANUP: cleanup,
    147    CommandType.CREATE_TABLES: create_tables,
    148    CommandType.MIGRATE_TABLES: migrate_tables,
    149}

class CommandType(builtins.str, enum.Enum): View Source

    18class CommandType(str, Enum):
    19    EVALUATE = "evaluate"
    20    PROMOTE = "promote"
    21    DEMOTE = "demote"
    22    CLEANUP = "cleanup"
    23    CREATE_TABLES = "create_tables"
    24    MIGRATE_TABLES = "migrate_tables"
    25
    26    # This makes it easy to integrate with argparse
    27    def __str__(self) -> str:
    28        return self.value

An enumeration.

EVALUATE = <CommandType.EVALUATE: 'evaluate'>
PROMOTE = <CommandType.PROMOTE: 'promote'>
DEMOTE = <CommandType.DEMOTE: 'demote'>
CLEANUP = <CommandType.CLEANUP: 'cleanup'>
CREATE_TABLES = <CommandType.CREATE_TABLES: 'create_tables'>
MIGRATE_TABLES = <CommandType.MIGRATE_TABLES: 'migrate_tables'>

##### Inherited Members

enum.Enum
  name
  value
builtins.str
  encode
  replace
  split
  rsplit
  join
  capitalize
  casefold
  title
  center
  count
  expandtabs
  find
  partition
  index
  ljust
  lower
  lstrip
  rfind
  rindex
  rjust
  rstrip
  rpartition
  splitlines
  strip
  swapcase
  translate
  upper
  startswith
  endswith
  removeprefix
  removesuffix
  isascii
  islower
  isupper
  istitle
  isspace
  isdecimal
  isdigit
  isnumeric
  isalpha
  isalnum
  isidentifier
  isprintable
  zfill
  format
  format_map
  maketrans

class EvaluateCommandPayload(sqlmesh.utils.pydantic.PydanticModel): View Source

    31class EvaluateCommandPayload(PydanticModel):
    32    snapshot: Snapshot
    33    parent_snapshots: t.Dict[str, Snapshot]
    34    start: TimeLike
    35    end: TimeLike
    36    latest: TimeLike
    37    is_dev: bool

##### Inherited Members

pydantic.main.BaseModel
  BaseModel
  parse_obj
  parse_raw
  parse_file
  from_orm
  construct
  copy
  schema
  schema_json
  validate
  update_forward_refs
sqlmesh.utils.pydantic.PydanticModel
  Config
  dict
  json
  missing_required_fields
  extra_fields
  all_fields
  required_fields

class PromoteCommandPayload(sqlmesh.utils.pydantic.PydanticModel): View Source

    40class PromoteCommandPayload(PydanticModel):
    41    snapshots: t.List[SnapshotTableInfo]
    42    environment: str
    43    is_dev: bool

##### Inherited Members

pydantic.main.BaseModel
  BaseModel
  parse_obj
  parse_raw
  parse_file
  from_orm
  construct
  copy
  schema
  schema_json
  validate
  update_forward_refs
sqlmesh.utils.pydantic.PydanticModel
  Config
  dict
  json
  missing_required_fields
  extra_fields
  all_fields
  required_fields

class DemoteCommandPayload(sqlmesh.utils.pydantic.PydanticModel): View Source

    46class DemoteCommandPayload(PydanticModel):
    47    snapshots: t.List[SnapshotTableInfo]
    48    environment: str

##### Inherited Members

pydantic.main.BaseModel
  BaseModel
  parse_obj
  parse_raw
  parse_file
  from_orm
  construct
  copy
  schema
  schema_json
  validate
  update_forward_refs
sqlmesh.utils.pydantic.PydanticModel
  Config
  dict
  json
  missing_required_fields
  extra_fields
  all_fields
  required_fields

class CleanupCommandPayload(sqlmesh.utils.pydantic.PydanticModel): View Source

    51class CleanupCommandPayload(PydanticModel):
    52    environments: t.List[Environment]
    53    snapshots: t.List[SnapshotTableInfo]

##### Inherited Members

pydantic.main.BaseModel
  BaseModel
  parse_obj
  parse_raw
  parse_file
  from_orm
  construct
  copy
  schema
  schema_json
  validate
  update_forward_refs
sqlmesh.utils.pydantic.PydanticModel
  Config
  dict
  json
  missing_required_fields
  extra_fields
  all_fields
  required_fields

class CreateTablesCommandPayload(sqlmesh.utils.pydantic.PydanticModel): View Source

    56class CreateTablesCommandPayload(PydanticModel):
    57    target_snapshot_ids: t.List[SnapshotId]
    58    snapshots: t.List[Snapshot]

##### Inherited Members

pydantic.main.BaseModel
  BaseModel
  parse_obj
  parse_raw
  parse_file
  from_orm
  construct
  copy
  schema
  schema_json
  validate
  update_forward_refs
sqlmesh.utils.pydantic.PydanticModel
  Config
  dict
  json
  missing_required_fields
  extra_fields
  all_fields
  required_fields

class MigrateTablesCommandPayload(sqlmesh.utils.pydantic.PydanticModel): View Source

    61class MigrateTablesCommandPayload(PydanticModel):
    62    snapshots: t.List[SnapshotTableInfo]

##### Inherited Members

pydantic.main.BaseModel
  BaseModel
  parse_obj
  parse_raw
  parse_file
  from_orm
  construct
  copy
  schema
  schema_json
  validate
  update_forward_refs
sqlmesh.utils.pydantic.PydanticModel
  Config
  dict
  json
  missing_required_fields
  extra_fields
  all_fields
  required_fields

def evaluate( evaluator: sqlmesh.core.snapshot.evaluator.SnapshotEvaluator, command_payload: Union[str, sqlmesh.engines.commands.EvaluateCommandPayload]) -> None: View Source

    65def evaluate(
    66    evaluator: SnapshotEvaluator, command_payload: t.Union[str, EvaluateCommandPayload]
    67) -> None:
    68    if isinstance(command_payload, str):
    69        command_payload = EvaluateCommandPayload.parse_raw(command_payload)
    70
    71    parent_snapshots = command_payload.parent_snapshots
    72    parent_snapshots[command_payload.snapshot.name] = command_payload.snapshot
    73
    74    evaluator.evaluate(
    75        command_payload.snapshot,
    76        command_payload.start,
    77        command_payload.end,
    78        command_payload.latest,
    79        snapshots=parent_snapshots,
    80        is_dev=command_payload.is_dev,
    81    )
    82    evaluator.audit(
    83        snapshot=command_payload.snapshot,
    84        start=command_payload.start,
    85        end=command_payload.end,
    86        latest=command_payload.latest,
    87        snapshots=parent_snapshots,
    88        is_dev=command_payload.is_dev,
    89    )

def promote( evaluator: sqlmesh.core.snapshot.evaluator.SnapshotEvaluator, command_payload: Union[str, sqlmesh.engines.commands.PromoteCommandPayload]) -> None: View Source

     92def promote(
     93    evaluator: SnapshotEvaluator, command_payload: t.Union[str, PromoteCommandPayload]
     94) -> None:
     95    if isinstance(command_payload, str):
     96        command_payload = PromoteCommandPayload.parse_raw(command_payload)
     97    evaluator.promote(
     98        command_payload.snapshots,
     99        command_payload.environment,
    100        is_dev=command_payload.is_dev,
    101    )

def demote( evaluator: sqlmesh.core.snapshot.evaluator.SnapshotEvaluator, command_payload: Union[str, sqlmesh.engines.commands.DemoteCommandPayload]) -> None: View Source

    104def demote(
    105    evaluator: SnapshotEvaluator, command_payload: t.Union[str, DemoteCommandPayload]
    106) -> None:
    107    if isinstance(command_payload, str):
    108        command_payload = DemoteCommandPayload.parse_raw(command_payload)
    109    evaluator.demote(command_payload.snapshots, command_payload.environment)

def cleanup( evaluator: sqlmesh.core.snapshot.evaluator.SnapshotEvaluator, command_payload: Union[str, sqlmesh.engines.commands.CleanupCommandPayload]) -> None: View Source

    112def cleanup(
    113    evaluator: SnapshotEvaluator, command_payload: t.Union[str, CleanupCommandPayload]
    114) -> None:
    115    if isinstance(command_payload, str):
    116        command_payload = CleanupCommandPayload.parse_raw(command_payload)
    117    for environment in command_payload.environments:
    118        evaluator.demote(environment.snapshots, environment.name)
    119    evaluator.cleanup(command_payload.snapshots)

def create_tables( evaluator: sqlmesh.core.snapshot.evaluator.SnapshotEvaluator, command_payload: Union[str, sqlmesh.engines.commands.CreateTablesCommandPayload]) -> None: View Source

    122def create_tables(
    123    evaluator: SnapshotEvaluator,
    124    command_payload: t.Union[str, CreateTablesCommandPayload],
    125) -> None:
    126    if isinstance(command_payload, str):
    127        command_payload = CreateTablesCommandPayload.parse_raw(command_payload)
    128
    129    snapshots_by_id = {s.snapshot_id: s for s in command_payload.snapshots}
    130    target_snapshots = [snapshots_by_id[sid] for sid in command_payload.target_snapshot_ids]
    131    evaluator.create(target_snapshots, snapshots_by_id)

def migrate_tables( evaluator: sqlmesh.core.snapshot.evaluator.SnapshotEvaluator, command_payload: Union[str, sqlmesh.engines.commands.MigrateTablesCommandPayload]) -> None: View Source

    134def migrate_tables(
    135    evaluator: SnapshotEvaluator,
    136    command_payload: t.Union[str, MigrateTablesCommandPayload],
    137) -> None:
    138    if isinstance(command_payload, str):
    139        command_payload = MigrateTablesCommandPayload.parse_raw(command_payload)
    140    evaluator.migrate(command_payload.snapshots)