sqlmesh.dbt.source API documentation

Edit on GitHub

# sqlmesh.dbt.source

View Source

     1from __future__ import annotations
     2
     3import typing as t
     4
     5from dbt.contracts.relation import RelationType
     6from pydantic import Field, validator
     7
     8from sqlmesh.core.config.base import UpdateStrategy
     9from sqlmesh.dbt.column import ColumnConfig, yaml_to_columns
    10from sqlmesh.dbt.common import GeneralConfig
    11from sqlmesh.utils.conversions import ensure_bool
    12
    13
    14class SourceConfig(GeneralConfig):
    15    """
    16    Args:
    17        config_name: The schema.table_name names declared in source config
    18        name: The name of the source or table
    19        database: Name of the database where the table is stored. By default, the project's target database is used.
    20        schema: The scehma name as stored in the database. If not specified, the source name is used.
    21        identifier: The table name as stored in the database. If not specified, the source table name is used
    22        loader: Describes the tool that loads the source into the warehouse
    23        overrides: Override a source defined in the specified package
    24        freshness: Dictionary specifying maximum time, since the most recent record, to consider the source fresh
    25        loaded_at_field: Column name or expression that returns a timestamp indicating freshness
    26        quoting: Dictionary of what to quote (database, schema, identifier) when resolving the source() method
    27        external: Dictionary of metadata properties specific to sources that point to external tables
    28        columns: Columns within the source
    29    """
    30
    31    # sqlmesh fields
    32    config_name: str = ""
    33
    34    # DBT configuration fields
    35    name: t.Optional[str] = None
    36    database: t.Optional[str] = None
    37    schema_: t.Optional[str] = Field(None, alias="schema")
    38    identifier: t.Optional[str] = None
    39    loader: t.Optional[str] = None
    40    overrides: t.Optional[str] = None
    41    freshness: t.Optional[t.Dict[str, t.Any]] = {}
    42    loaded_at_field: t.Optional[str] = None
    43    quoting: t.Optional[t.Dict[str, bool]] = {}
    44    external: t.Optional[t.Dict[str, t.Any]] = {}
    45    columns: t.Dict[str, ColumnConfig] = {}
    46
    47    @validator("quoting", pre=True)
    48    def _validate_quoting(cls, v: t.Dict[str, t.Any]) -> t.Dict[str, bool]:
    49        return {key: ensure_bool(val) for key, val in v.items()}
    50
    51    @validator("columns", pre=True)
    52    def _validate_columns(cls, v: t.Any) -> t.Dict[str, ColumnConfig]:
    53        return yaml_to_columns(v)
    54
    55    _FIELD_UPDATE_STRATEGY: t.ClassVar[t.Dict[str, UpdateStrategy]] = {
    56        **GeneralConfig._FIELD_UPDATE_STRATEGY,
    57        **{"columns": UpdateStrategy.KEY_EXTEND},
    58    }
    59
    60    @property
    61    def table_name(self) -> t.Optional[str]:
    62        return self.identifier or self.name
    63
    64    @property
    65    def source_name(self) -> str:
    66        return ".".join(part for part in (self.schema_, self.table_name) if part)
    67
    68    @property
    69    def relation_info(self) -> t.Dict[str, t.Any]:
    70        return {
    71            "database": self.database,
    72            "schema": self.schema_,
    73            "identifier": self.table_name,
    74            "type": RelationType.External.value,
    75        }

class SourceConfig(sqlmesh.dbt.common.GeneralConfig): View Source

    15class SourceConfig(GeneralConfig):
    16    """
    17    Args:
    18        config_name: The schema.table_name names declared in source config
    19        name: The name of the source or table
    20        database: Name of the database where the table is stored. By default, the project's target database is used.
    21        schema: The scehma name as stored in the database. If not specified, the source name is used.
    22        identifier: The table name as stored in the database. If not specified, the source table name is used
    23        loader: Describes the tool that loads the source into the warehouse
    24        overrides: Override a source defined in the specified package
    25        freshness: Dictionary specifying maximum time, since the most recent record, to consider the source fresh
    26        loaded_at_field: Column name or expression that returns a timestamp indicating freshness
    27        quoting: Dictionary of what to quote (database, schema, identifier) when resolving the source() method
    28        external: Dictionary of metadata properties specific to sources that point to external tables
    29        columns: Columns within the source
    30    """
    31
    32    # sqlmesh fields
    33    config_name: str = ""
    34
    35    # DBT configuration fields
    36    name: t.Optional[str] = None
    37    database: t.Optional[str] = None
    38    schema_: t.Optional[str] = Field(None, alias="schema")
    39    identifier: t.Optional[str] = None
    40    loader: t.Optional[str] = None
    41    overrides: t.Optional[str] = None
    42    freshness: t.Optional[t.Dict[str, t.Any]] = {}
    43    loaded_at_field: t.Optional[str] = None
    44    quoting: t.Optional[t.Dict[str, bool]] = {}
    45    external: t.Optional[t.Dict[str, t.Any]] = {}
    46    columns: t.Dict[str, ColumnConfig] = {}
    47
    48    @validator("quoting", pre=True)
    49    def _validate_quoting(cls, v: t.Dict[str, t.Any]) -> t.Dict[str, bool]:
    50        return {key: ensure_bool(val) for key, val in v.items()}
    51
    52    @validator("columns", pre=True)
    53    def _validate_columns(cls, v: t.Any) -> t.Dict[str, ColumnConfig]:
    54        return yaml_to_columns(v)
    55
    56    _FIELD_UPDATE_STRATEGY: t.ClassVar[t.Dict[str, UpdateStrategy]] = {
    57        **GeneralConfig._FIELD_UPDATE_STRATEGY,
    58        **{"columns": UpdateStrategy.KEY_EXTEND},
    59    }
    60
    61    @property
    62    def table_name(self) -> t.Optional[str]:
    63        return self.identifier or self.name
    64
    65    @property
    66    def source_name(self) -> str:
    67        return ".".join(part for part in (self.schema_, self.table_name) if part)
    68
    69    @property
    70    def relation_info(self) -> t.Dict[str, t.Any]:
    71        return {
    72            "database": self.database,
    73            "schema": self.schema_,
    74            "identifier": self.table_name,
    75            "type": RelationType.External.value,
    76        }

###### Arguments:

- config_name: The schema.table_name names declared in source config
- name: The name of the source or table
- database: Name of the database where the table is stored. By default, the project's target database is used.
- schema: The scehma name as stored in the database. If not specified, the source name is used.
- identifier: The table name as stored in the database. If not specified, the source table name is used
- loader: Describes the tool that loads the source into the warehouse
- overrides: Override a source defined in the specified package
- freshness: Dictionary specifying maximum time, since the most recent record, to consider the source fresh
- loaded_at_field: Column name or expression that returns a timestamp indicating freshness
- quoting: Dictionary of what to quote (database, schema, identifier) when resolving the source() method
- external: Dictionary of metadata properties specific to sources that point to external tables
- columns: Columns within the source

##### Inherited Members

pydantic.main.BaseModel
  BaseModel
  parse_obj
  parse_raw
  parse_file
  from_orm
  construct
  copy
  schema
  schema_json
  validate
  update_forward_refs
sqlmesh.dbt.common.GeneralConfig
  replace
  render_config
sqlmesh.dbt.common.DbtConfig
  Config
sqlmesh.core.config.base.BaseConfig
  update_with
sqlmesh.utils.pydantic.PydanticModel
  dict
  json
  missing_required_fields
  extra_fields
  all_fields
  required_fields