sqlmesh.core.schema_diff API documentation

Edit on GitHub

# sqlmesh.core.schema_diff

View Source

     1from __future__ import annotations
     2
     3import typing as t
     4from enum import Enum, auto
     5
     6from sqlmesh.utils.pydantic import PydanticModel
     7
     8if t.TYPE_CHECKING:
     9    from sqlmesh.core.engine_adapter import EngineAdapter
    10
    11
    12class SchemaDeltaOp(Enum):
    13    ADD = auto()
    14    DROP = auto()
    15    ALTER_TYPE = auto()
    16
    17
    18class SchemaDelta(PydanticModel):
    19    column_name: str
    20    column_type: str
    21    op: SchemaDeltaOp
    22
    23    @classmethod
    24    def add(cls, column_name: str, column_type: str) -> SchemaDelta:
    25        return cls(column_name=column_name, column_type=column_type, op=SchemaDeltaOp.ADD)
    26
    27    @classmethod
    28    def drop(cls, column_name: str, column_type: str) -> SchemaDelta:
    29        return cls(column_name=column_name, column_type=column_type, op=SchemaDeltaOp.DROP)
    30
    31    @classmethod
    32    def alter_type(cls, column_name: str, column_type: str) -> SchemaDelta:
    33        return cls(
    34            column_name=column_name,
    35            column_type=column_type,
    36            op=SchemaDeltaOp.ALTER_TYPE,
    37        )
    38
    39
    40class SchemaDiffCalculator:
    41    """Calculates the difference between table schemas.
    42
    43    Args:
    44        engine_adapter: The engine adapter.
    45        is_type_transition_allowed: The predicate which accepts the source type and the target type
    46            and returns True if the transition from source to target is allowed and False otherwise.
    47            Default: no type transitions are allowed.
    48    """
    49
    50    def __init__(
    51        self,
    52        engine_adapter: EngineAdapter,
    53        is_type_transition_allowed: t.Optional[t.Callable[[str, str], bool]] = None,
    54    ):
    55        self.engine_adapter = engine_adapter
    56        self.is_type_transition_allowed = is_type_transition_allowed or (lambda src, tgt: False)
    57
    58    def calculate(self, apply_to_table: str, schema_from_table: str) -> t.List[SchemaDelta]:
    59        """Calculates a list of schema deltas between the two tables, applying which in order to the first table
    60        brings its schema in correspondence with the schema of the second table.
    61
    62        Changes in positions of otherwise unchanged columns are currently ignored and are not reflected in the output.
    63
    64        Additionally the implementation currently doesn't differentiate between regular columns and partition ones.
    65        It's a responsibility of a caller to determine whether a returned operation is allowed on partition columns or not.
    66
    67        Args:
    68            apply_to_table: The name of the table to which deltas will be applied.
    69            schema_from_table: The schema of this table will be used for comparison.
    70
    71        Returns:
    72            The list of deltas.
    73        """
    74        to_schema = self.engine_adapter.columns(apply_to_table)
    75        from_schema = self.engine_adapter.columns(schema_from_table)
    76
    77        result = []
    78
    79        for to_column_name, to_column_type in to_schema.items():
    80            from_column_type = from_schema.get(to_column_name)
    81            from_column_type = from_column_type.upper() if from_column_type else None
    82            if from_column_type != to_column_type.upper():
    83                if from_column_type and self.is_type_transition_allowed(
    84                    to_column_type, from_column_type
    85                ):
    86                    result.append(SchemaDelta.alter_type(to_column_name, from_column_type))
    87                else:
    88                    result.append(SchemaDelta.drop(to_column_name, to_column_type))
    89                    if from_column_type:
    90                        result.append(SchemaDelta.add(to_column_name, from_column_type))
    91
    92        for from_column_name, from_column_type in from_schema.items():
    93            if from_column_name not in to_schema:
    94                result.append(SchemaDelta.add(from_column_name, from_column_type))
    95
    96        return result

class SchemaDeltaOp(enum.Enum): View Source

    13class SchemaDeltaOp(Enum):
    14    ADD = auto()
    15    DROP = auto()
    16    ALTER_TYPE = auto()

An enumeration.

ADD = <SchemaDeltaOp.ADD: 1>
DROP = <SchemaDeltaOp.DROP: 2>
ALTER_TYPE = <SchemaDeltaOp.ALTER_TYPE: 3>

##### Inherited Members

enum.Enum
  name
  value

class SchemaDelta(sqlmesh.utils.pydantic.PydanticModel): View Source

    19class SchemaDelta(PydanticModel):
    20    column_name: str
    21    column_type: str
    22    op: SchemaDeltaOp
    23
    24    @classmethod
    25    def add(cls, column_name: str, column_type: str) -> SchemaDelta:
    26        return cls(column_name=column_name, column_type=column_type, op=SchemaDeltaOp.ADD)
    27
    28    @classmethod
    29    def drop(cls, column_name: str, column_type: str) -> SchemaDelta:
    30        return cls(column_name=column_name, column_type=column_type, op=SchemaDeltaOp.DROP)
    31
    32    @classmethod
    33    def alter_type(cls, column_name: str, column_type: str) -> SchemaDelta:
    34        return cls(
    35            column_name=column_name,
    36            column_type=column_type,
    37            op=SchemaDeltaOp.ALTER_TYPE,
    38        )

@classmethod
def add( cls, column_name: str, column_type: str) -> sqlmesh.core.schema_diff.SchemaDelta: View Source

    24    @classmethod
    25    def add(cls, column_name: str, column_type: str) -> SchemaDelta:
    26        return cls(column_name=column_name, column_type=column_type, op=SchemaDeltaOp.ADD)

@classmethod
def drop( cls, column_name: str, column_type: str) -> sqlmesh.core.schema_diff.SchemaDelta: View Source

    28    @classmethod
    29    def drop(cls, column_name: str, column_type: str) -> SchemaDelta:
    30        return cls(column_name=column_name, column_type=column_type, op=SchemaDeltaOp.DROP)

@classmethod
def alter_type( cls, column_name: str, column_type: str) -> sqlmesh.core.schema_diff.SchemaDelta: View Source

    32    @classmethod
    33    def alter_type(cls, column_name: str, column_type: str) -> SchemaDelta:
    34        return cls(
    35            column_name=column_name,
    36            column_type=column_type,
    37            op=SchemaDeltaOp.ALTER_TYPE,
    38        )

##### Inherited Members

pydantic.main.BaseModel
  BaseModel
  parse_obj
  parse_raw
  parse_file
  from_orm
  construct
  copy
  schema
  schema_json
  validate
  update_forward_refs
sqlmesh.utils.pydantic.PydanticModel
  Config
  dict
  json
  missing_required_fields
  extra_fields
  all_fields
  required_fields

class SchemaDiffCalculator: View Source

    41class SchemaDiffCalculator:
    42    """Calculates the difference between table schemas.
    43
    44    Args:
    45        engine_adapter: The engine adapter.
    46        is_type_transition_allowed: The predicate which accepts the source type and the target type
    47            and returns True if the transition from source to target is allowed and False otherwise.
    48            Default: no type transitions are allowed.
    49    """
    50
    51    def __init__(
    52        self,
    53        engine_adapter: EngineAdapter,
    54        is_type_transition_allowed: t.Optional[t.Callable[[str, str], bool]] = None,
    55    ):
    56        self.engine_adapter = engine_adapter
    57        self.is_type_transition_allowed = is_type_transition_allowed or (lambda src, tgt: False)
    58
    59    def calculate(self, apply_to_table: str, schema_from_table: str) -> t.List[SchemaDelta]:
    60        """Calculates a list of schema deltas between the two tables, applying which in order to the first table
    61        brings its schema in correspondence with the schema of the second table.
    62
    63        Changes in positions of otherwise unchanged columns are currently ignored and are not reflected in the output.
    64
    65        Additionally the implementation currently doesn't differentiate between regular columns and partition ones.
    66        It's a responsibility of a caller to determine whether a returned operation is allowed on partition columns or not.
    67
    68        Args:
    69            apply_to_table: The name of the table to which deltas will be applied.
    70            schema_from_table: The schema of this table will be used for comparison.
    71
    72        Returns:
    73            The list of deltas.
    74        """
    75        to_schema = self.engine_adapter.columns(apply_to_table)
    76        from_schema = self.engine_adapter.columns(schema_from_table)
    77
    78        result = []
    79
    80        for to_column_name, to_column_type in to_schema.items():
    81            from_column_type = from_schema.get(to_column_name)
    82            from_column_type = from_column_type.upper() if from_column_type else None
    83            if from_column_type != to_column_type.upper():
    84                if from_column_type and self.is_type_transition_allowed(
    85                    to_column_type, from_column_type
    86                ):
    87                    result.append(SchemaDelta.alter_type(to_column_name, from_column_type))
    88                else:
    89                    result.append(SchemaDelta.drop(to_column_name, to_column_type))
    90                    if from_column_type:
    91                        result.append(SchemaDelta.add(to_column_name, from_column_type))
    92
    93        for from_column_name, from_column_type in from_schema.items():
    94            if from_column_name not in to_schema:
    95                result.append(SchemaDelta.add(from_column_name, from_column_type))
    96
    97        return result

Calculates the difference between table schemas.

###### Arguments:

- engine_adapter: The engine adapter.
- is_type_transition_allowed: The predicate which accepts the source type and the target type and returns True if the transition from source to target is allowed and False otherwise. Default: no type transitions are allowed.

SchemaDiffCalculator( engine_adapter: sqlmesh.core.engine_adapter.base.EngineAdapter, is_type_transition_allowed: Optional[Callable[[str, str], bool]] = None) View Source

    51    def __init__(
    52        self,
    53        engine_adapter: EngineAdapter,
    54        is_type_transition_allowed: t.Optional[t.Callable[[str, str], bool]] = None,
    55    ):
    56        self.engine_adapter = engine_adapter
    57        self.is_type_transition_allowed = is_type_transition_allowed or (lambda src, tgt: False)

def calculate( self, apply_to_table: str, schema_from_table: str) -> List[sqlmesh.core.schema_diff.SchemaDelta]: View Source

    59    def calculate(self, apply_to_table: str, schema_from_table: str) -> t.List[SchemaDelta]:
    60        """Calculates a list of schema deltas between the two tables, applying which in order to the first table
    61        brings its schema in correspondence with the schema of the second table.
    62
    63        Changes in positions of otherwise unchanged columns are currently ignored and are not reflected in the output.
    64
    65        Additionally the implementation currently doesn't differentiate between regular columns and partition ones.
    66        It's a responsibility of a caller to determine whether a returned operation is allowed on partition columns or not.
    67
    68        Args:
    69            apply_to_table: The name of the table to which deltas will be applied.
    70            schema_from_table: The schema of this table will be used for comparison.
    71
    72        Returns:
    73            The list of deltas.
    74        """
    75        to_schema = self.engine_adapter.columns(apply_to_table)
    76        from_schema = self.engine_adapter.columns(schema_from_table)
    77
    78        result = []
    79
    80        for to_column_name, to_column_type in to_schema.items():
    81            from_column_type = from_schema.get(to_column_name)
    82            from_column_type = from_column_type.upper() if from_column_type else None
    83            if from_column_type != to_column_type.upper():
    84                if from_column_type and self.is_type_transition_allowed(
    85                    to_column_type, from_column_type
    86                ):
    87                    result.append(SchemaDelta.alter_type(to_column_name, from_column_type))
    88                else:
    89                    result.append(SchemaDelta.drop(to_column_name, to_column_type))
    90                    if from_column_type:
    91                        result.append(SchemaDelta.add(to_column_name, from_column_type))
    92
    93        for from_column_name, from_column_type in from_schema.items():
    94            if from_column_name not in to_schema:
    95                result.append(SchemaDelta.add(from_column_name, from_column_type))
    96
    97        return result

Calculates a list of schema deltas between the two tables, applying which in order to the first table brings its schema in correspondence with the schema of the second table.

Changes in positions of otherwise unchanged columns are currently ignored and are not reflected in the output.

Additionally the implementation currently doesn't differentiate between regular columns and partition ones. It's a responsibility of a caller to determine whether a returned operation is allowed on partition columns or not.

###### Arguments:

- apply_to_table: The name of the table to which deltas will be applied.
- schema_from_table: The schema of this table will be used for comparison.

###### Returns:

> The list of deltas.