sqlmesh.dbt.basemodel API documentation

Edit on GitHub

# sqlmesh.dbt.basemodel

View Source

      1from __future__ import annotations
      2
      3import typing as t
      4from abc import abstractmethod
      5from enum import Enum
      6from pathlib import Path
      7
      8from dbt.adapters.base import BaseRelation
      9from dbt.contracts.relation import RelationType
     10from pydantic import Field, validator
     11from sqlglot.helper import ensure_list
     12
     13from sqlmesh.core import constants as c
     14from sqlmesh.core import dialect as d
     15from sqlmesh.core.config.base import UpdateStrategy
     16from sqlmesh.core.model import Model
     17from sqlmesh.dbt.adapter import ParsetimeAdapter
     18from sqlmesh.dbt.builtin import create_builtin_globals
     19from sqlmesh.dbt.column import (
     20    ColumnConfig,
     21    column_descriptions_to_sqlmesh,
     22    column_types_to_sqlmesh,
     23    yaml_to_columns,
     24)
     25from sqlmesh.dbt.common import DbtContext, GeneralConfig, SqlStr
     26from sqlmesh.utils import AttributeDict
     27from sqlmesh.utils.conversions import ensure_bool
     28from sqlmesh.utils.date import date_dict
     29from sqlmesh.utils.errors import ConfigError
     30from sqlmesh.utils.jinja import MacroReference, extract_macro_references
     31from sqlmesh.utils.pydantic import PydanticModel
     32
     33BMC = t.TypeVar("BMC", bound="BaseModelConfig")
     34
     35
     36class Dependencies(PydanticModel):
     37    """
     38    DBT dependencies for a model, macro, etc.
     39
     40    Args:
     41        macros: The references to macros
     42        sources: The "source_name.table_name" for source tables used
     43        refs: The table_name for models used
     44        variables: The names of variables used, mapped to a flag that indicates whether their
     45            definition is optional or not.
     46    """
     47
     48    macros: t.Set[MacroReference] = set()
     49    sources: t.Set[str] = set()
     50    refs: t.Set[str] = set()
     51    variables: t.Set[str] = set()
     52
     53    def union(self, other: Dependencies) -> Dependencies:
     54        dependencies = Dependencies()
     55        dependencies.macros = self.macros | other.macros
     56        dependencies.sources = self.sources | other.sources
     57        dependencies.refs = self.refs | other.refs
     58        dependencies.variables = self.variables | other.variables
     59
     60        return dependencies
     61
     62
     63class Materialization(str, Enum):
     64    """DBT model materializations"""
     65
     66    TABLE = "table"
     67    VIEW = "view"
     68    INCREMENTAL = "incremental"
     69    EPHEMERAL = "ephemeral"
     70
     71
     72class BaseModelConfig(GeneralConfig):
     73    """
     74    Args:
     75        owner: The owner of the model.
     76        stamp: An optional arbitrary string sequence used to create new model versions without making
     77            changes to any of the functional components of the definition.
     78        storage_format: The storage format used to store the physical table, only applicable in certain engines.
     79            (eg. 'parquet')
     80        path: The file path of the model
     81        target_schema: The schema for the profile target
     82        database: Database the model is stored in
     83        schema: Custom schema name added to the model schema name
     84        alias: Relation identifier for this model instead of the filename
     85        pre-hook: List of SQL statements to run before the model is built
     86        post-hook: List of SQL statements to run after the model is built
     87        full_refresh: Forces the model to always do a full refresh or never do a full refresh
     88        grants: Set or revoke permissions to the database object for this model
     89        columns: Column information for the model
     90    """
     91
     92    # sqlmesh fields
     93    owner: t.Optional[str] = None
     94    stamp: t.Optional[str] = None
     95    storage_format: t.Optional[str] = None
     96    path: Path = Path()
     97    target_schema: str = ""
     98    _dependencies: Dependencies = Dependencies()
     99    _variables: t.Dict[str, bool] = {}
    100
    101    # DBT configuration fields
    102    database: t.Optional[str] = None
    103    schema_: t.Optional[str] = Field(None, alias="schema")
    104    alias: t.Optional[str] = None
    105    pre_hook: t.List[SqlStr] = Field([], alias="pre-hook")
    106    post_hook: t.List[SqlStr] = Field([], alias="post-hook")
    107    full_refresh: t.Optional[bool] = None
    108    grants: t.Dict[str, t.List[str]] = {}
    109    columns: t.Dict[str, ColumnConfig] = {}
    110
    111    @validator("pre_hook", "post_hook", pre=True)
    112    def _validate_hooks(cls, v: t.Union[str, t.List[t.Union[SqlStr, str]]]) -> t.List[SqlStr]:
    113        return [SqlStr(val) for val in ensure_list(v)]
    114
    115    @validator("full_refresh", pre=True)
    116    def _validate_bool(cls, v: str) -> bool:
    117        return ensure_bool(v)
    118
    119    @validator("grants", pre=True)
    120    def _validate_grants(cls, v: t.Dict[str, str]) -> t.Dict[str, t.List[str]]:
    121        return {key: ensure_list(value) for key, value in v.items()}
    122
    123    @validator("columns", pre=True)
    124    def _validate_columns(cls, v: t.Any) -> t.Dict[str, ColumnConfig]:
    125        if not isinstance(v, dict) or all(isinstance(col, ColumnConfig) for col in v.values()):
    126            return v
    127
    128        return yaml_to_columns(v)
    129
    130    _FIELD_UPDATE_STRATEGY: t.ClassVar[t.Dict[str, UpdateStrategy]] = {
    131        **GeneralConfig._FIELD_UPDATE_STRATEGY,
    132        **{
    133            "grants": UpdateStrategy.KEY_EXTEND,
    134            "path": UpdateStrategy.IMMUTABLE,
    135            "pre-hook": UpdateStrategy.EXTEND,
    136            "post-hook": UpdateStrategy.EXTEND,
    137            "columns": UpdateStrategy.KEY_EXTEND,
    138        },
    139    }
    140
    141    @property
    142    def all_sql(self) -> SqlStr:
    143        return SqlStr("")
    144
    145    @property
    146    def table_schema(self) -> str:
    147        """
    148        Get the full schema name
    149        """
    150        return "_".join(part for part in (self.target_schema, self.schema_) if part)
    151
    152    @property
    153    def table_name(self) -> str:
    154        """
    155        Get the table name
    156        """
    157        return self.alias or self.path.stem
    158
    159    @property
    160    def model_name(self) -> str:
    161        """
    162        Get the sqlmesh model name
    163
    164        Returns:
    165            The sqlmesh model name
    166        """
    167        return ".".join(
    168            part for part in (self.database, self.table_schema, self.table_name) if part
    169        )
    170
    171    @property
    172    def model_materialization(self) -> Materialization:
    173        return Materialization.TABLE
    174
    175    @property
    176    def relation_info(self) -> AttributeDict[str, t.Any]:
    177        if self.model_materialization == Materialization.VIEW:
    178            relation_type = RelationType.View
    179        elif self.model_materialization == Materialization.EPHEMERAL:
    180            relation_type = RelationType.CTE
    181        else:
    182            relation_type = RelationType.Table
    183
    184        return AttributeDict(
    185            {
    186                "database": self.database,
    187                "schema": self.table_schema,
    188                "identifier": self.table_name,
    189                "type": relation_type.value,
    190            }
    191        )
    192
    193    def sqlmesh_model_kwargs(self, model_context: DbtContext) -> t.Dict[str, t.Any]:
    194        """Get common sqlmesh model parameters"""
    195        jinja_macros = model_context.jinja_macros.trim(self._dependencies.macros)
    196        jinja_macros.global_objs.update(
    197            {
    198                "this": self.relation_info,
    199                "schema": self.table_schema,
    200                **model_context.jinja_globals,  # type: ignore
    201            }
    202        )
    203
    204        optional_kwargs: t.Dict[str, t.Any] = {}
    205        for field in ("description", "owner", "stamp", "storage_format"):
    206            field_val = getattr(self, field, None) or self.meta.get(field, None)
    207            if field_val:
    208                optional_kwargs[field] = field_val
    209
    210        return {
    211            "columns": column_types_to_sqlmesh(self.columns) or None,
    212            "column_descriptions_": column_descriptions_to_sqlmesh(self.columns) or None,
    213            "depends_on": {model_context.refs[ref] for ref in self._dependencies.refs},
    214            "jinja_macros": jinja_macros,
    215            "path": self.path,
    216            "pre": [exp for hook in self.pre_hook for exp in d.parse(hook)],
    217            "post": [exp for hook in self.post_hook for exp in d.parse(hook)],
    218            **optional_kwargs,
    219        }
    220
    221    def render_config(self: BMC, context: DbtContext) -> BMC:
    222        rendered = super().render_config(context)
    223        rendered._dependencies = Dependencies(macros=extract_macro_references(rendered.all_sql))
    224        rendered = ModelSqlRenderer(context, rendered).enriched_config
    225
    226        rendered_dependencies = rendered._dependencies
    227        for dependency in rendered_dependencies.refs:
    228            model = context.models.get(dependency)
    229            if model and model.materialized == Materialization.EPHEMERAL:
    230                rendered._dependencies = rendered._dependencies.union(
    231                    model.render_config(context)._dependencies
    232                )
    233                rendered._dependencies.refs.discard(dependency)
    234
    235        return rendered
    236
    237    @abstractmethod
    238    def to_sqlmesh(self, context: DbtContext) -> Model:
    239        """Convert DBT model into sqlmesh Model"""
    240
    241    def _context_for_dependencies(
    242        self, context: DbtContext, dependencies: Dependencies
    243    ) -> DbtContext:
    244        model_context = context.copy()
    245
    246        model_context.sources = {
    247            name: value for name, value in context.sources.items() if name in dependencies.sources
    248        }
    249        model_context.seeds = {
    250            name: value for name, value in context.seeds.items() if name in dependencies.refs
    251        }
    252        model_context.models = {
    253            name: value for name, value in context.models.items() if name in dependencies.refs
    254        }
    255        model_context.variables = {
    256            name: value
    257            for name, value in context.variables.items()
    258            if name in dependencies.variables
    259        }
    260
    261        return model_context
    262
    263
    264class ModelSqlRenderer(t.Generic[BMC]):
    265    def __init__(self, context: DbtContext, config: BMC):
    266        self.context = context
    267        self.config = config
    268
    269        self._captured_dependencies: Dependencies = Dependencies()
    270        self._rendered_sql: t.Optional[str] = None
    271        self._enriched_config: BMC = config.copy()
    272
    273        self._jinja_globals = create_builtin_globals(
    274            jinja_macros=context.jinja_macros,
    275            jinja_globals={
    276                **context.jinja_globals,
    277                **date_dict(c.EPOCH, c.EPOCH, c.EPOCH),
    278                "config": self._config,
    279                "ref": self._ref,
    280                "var": self._var,
    281                "source": self._source,
    282                "this": self.config.relation_info,
    283                "schema": self.config.table_schema,
    284            },
    285            engine_adapter=None,
    286        )
    287
    288        # Set the adapter separately since it requires jinja globals to passed into it.
    289        self._jinja_globals["adapter"] = ModelSqlRenderer.TrackingAdapter(
    290            self,
    291            context.jinja_macros,
    292            jinja_globals=self._jinja_globals,
    293            dialect=context.engine_adapter.dialect if context.engine_adapter else "",
    294        )
    295
    296    @property
    297    def enriched_config(self) -> BMC:
    298        if self._rendered_sql is None:
    299            self.render()
    300            self._enriched_config._dependencies = self._enriched_config._dependencies.union(
    301                self._captured_dependencies
    302            )
    303        return self._enriched_config
    304
    305    def render(self) -> str:
    306        if self._rendered_sql is None:
    307            registry = self.context.jinja_macros
    308            self._rendered_sql = (
    309                registry.build_environment(**self._jinja_globals)
    310                .from_string(self.config.all_sql)
    311                .render()
    312            )
    313        return self._rendered_sql
    314
    315    def _ref(self, package_name: str, model_name: t.Optional[str] = None) -> BaseRelation:
    316        if package_name in self.context.models:
    317            relation = BaseRelation.create(**self.context.models[package_name].relation_info)
    318        elif package_name in self.context.seeds:
    319            relation = BaseRelation.create(**self.context.seeds[package_name].relation_info)
    320        else:
    321            raise ConfigError(
    322                f"Model '{package_name}' was not found for model '{self.config.table_name}'."
    323            )
    324        self._captured_dependencies.refs.add(package_name)
    325        return relation
    326
    327    def _var(self, name: str, default: t.Optional[str] = None) -> t.Any:
    328        if default is None and name not in self.context.variables:
    329            raise ConfigError(
    330                f"Variable '{name}' was not found for model '{self.config.table_name}'."
    331            )
    332        self._captured_dependencies.variables.add(name)
    333        return self.context.variables.get(name, default)
    334
    335    def _source(self, source_name: str, table_name: str) -> BaseRelation:
    336        full_name = ".".join([source_name, table_name])
    337        if full_name not in self.context.sources:
    338            raise ConfigError(
    339                f"Source '{full_name}' was not found for model '{self.config.table_name}'."
    340            )
    341        self._captured_dependencies.sources.add(full_name)
    342        return BaseRelation.create(**self.context.sources[full_name].relation_info)
    343
    344    def _config(self, *args: t.Any, **kwargs: t.Any) -> str:
    345        if args and isinstance(args[0], dict):
    346            self._enriched_config = self._enriched_config.update_with(args[0])
    347        if kwargs:
    348            self._enriched_config = self._enriched_config.update_with(kwargs)
    349        return ""
    350
    351    class TrackingAdapter(ParsetimeAdapter):
    352        def __init__(self, outer_self: ModelSqlRenderer, *args: t.Any, **kwargs: t.Any):
    353            super().__init__(*args, **kwargs)
    354            self.outer_self = outer_self
    355            self.context = outer_self.context
    356
    357        def dispatch(self, name: str, package: t.Optional[str] = None) -> t.Callable:
    358            macros = (
    359                self.context.jinja_macros.packages.get(package, {})
    360                if package is not None
    361                else self.context.jinja_macros.root_macros
    362            )
    363            for target_name in macros:
    364                if target_name.endswith(f"__{name}"):
    365                    self.outer_self._captured_dependencies.macros.add(
    366                        MacroReference(package=package, name=target_name)
    367                    )
    368            return super().dispatch(name, package=package)

class Dependencies(sqlmesh.utils.pydantic.PydanticModel): View Source

    37class Dependencies(PydanticModel):
    38    """
    39    DBT dependencies for a model, macro, etc.
    40
    41    Args:
    42        macros: The references to macros
    43        sources: The "source_name.table_name" for source tables used
    44        refs: The table_name for models used
    45        variables: The names of variables used, mapped to a flag that indicates whether their
    46            definition is optional or not.
    47    """
    48
    49    macros: t.Set[MacroReference] = set()
    50    sources: t.Set[str] = set()
    51    refs: t.Set[str] = set()
    52    variables: t.Set[str] = set()
    53
    54    def union(self, other: Dependencies) -> Dependencies:
    55        dependencies = Dependencies()
    56        dependencies.macros = self.macros | other.macros
    57        dependencies.sources = self.sources | other.sources
    58        dependencies.refs = self.refs | other.refs
    59        dependencies.variables = self.variables | other.variables
    60
    61        return dependencies

DBT dependencies for a model, macro, etc.

###### Arguments:

- macros: The references to macros
- sources: The "source_name.table_name" for source tables used
- refs: The table_name for models used
- variables: The names of variables used, mapped to a flag that indicates whether their definition is optional or not.

def union( self, other: sqlmesh.dbt.basemodel.Dependencies) -> sqlmesh.dbt.basemodel.Dependencies: View Source

    54    def union(self, other: Dependencies) -> Dependencies:
    55        dependencies = Dependencies()
    56        dependencies.macros = self.macros | other.macros
    57        dependencies.sources = self.sources | other.sources
    58        dependencies.refs = self.refs | other.refs
    59        dependencies.variables = self.variables | other.variables
    60
    61        return dependencies

##### Inherited Members

pydantic.main.BaseModel
  BaseModel
  parse_obj
  parse_raw
  parse_file
  from_orm
  construct
  copy
  schema
  schema_json
  validate
  update_forward_refs
sqlmesh.utils.pydantic.PydanticModel
  Config
  dict
  json
  missing_required_fields
  extra_fields
  all_fields
  required_fields

class Materialization(builtins.str, enum.Enum): View Source

    64class Materialization(str, Enum):
    65    """DBT model materializations"""
    66
    67    TABLE = "table"
    68    VIEW = "view"
    69    INCREMENTAL = "incremental"
    70    EPHEMERAL = "ephemeral"

DBT model materializations

TABLE = <Materialization.TABLE: 'table'>
VIEW = <Materialization.VIEW: 'view'>
INCREMENTAL = <Materialization.INCREMENTAL: 'incremental'>
EPHEMERAL = <Materialization.EPHEMERAL: 'ephemeral'>

##### Inherited Members

enum.Enum
  name
  value
builtins.str
  encode
  replace
  split
  rsplit
  join
  capitalize
  casefold
  title
  center
  count
  expandtabs
  find
  partition
  index
  ljust
  lower
  lstrip
  rfind
  rindex
  rjust
  rstrip
  rpartition
  splitlines
  strip
  swapcase
  translate
  upper
  startswith
  endswith
  removeprefix
  removesuffix
  isascii
  islower
  isupper
  istitle
  isspace
  isdecimal
  isdigit
  isnumeric
  isalpha
  isalnum
  isidentifier
  isprintable
  zfill
  format
  format_map
  maketrans

class BaseModelConfig(sqlmesh.dbt.common.GeneralConfig): View Source

     73class BaseModelConfig(GeneralConfig):
     74    """
     75    Args:
     76        owner: The owner of the model.
     77        stamp: An optional arbitrary string sequence used to create new model versions without making
     78            changes to any of the functional components of the definition.
     79        storage_format: The storage format used to store the physical table, only applicable in certain engines.
     80            (eg. 'parquet')
     81        path: The file path of the model
     82        target_schema: The schema for the profile target
     83        database: Database the model is stored in
     84        schema: Custom schema name added to the model schema name
     85        alias: Relation identifier for this model instead of the filename
     86        pre-hook: List of SQL statements to run before the model is built
     87        post-hook: List of SQL statements to run after the model is built
     88        full_refresh: Forces the model to always do a full refresh or never do a full refresh
     89        grants: Set or revoke permissions to the database object for this model
     90        columns: Column information for the model
     91    """
     92
     93    # sqlmesh fields
     94    owner: t.Optional[str] = None
     95    stamp: t.Optional[str] = None
     96    storage_format: t.Optional[str] = None
     97    path: Path = Path()
     98    target_schema: str = ""
     99    _dependencies: Dependencies = Dependencies()
    100    _variables: t.Dict[str, bool] = {}
    101
    102    # DBT configuration fields
    103    database: t.Optional[str] = None
    104    schema_: t.Optional[str] = Field(None, alias="schema")
    105    alias: t.Optional[str] = None
    106    pre_hook: t.List[SqlStr] = Field([], alias="pre-hook")
    107    post_hook: t.List[SqlStr] = Field([], alias="post-hook")
    108    full_refresh: t.Optional[bool] = None
    109    grants: t.Dict[str, t.List[str]] = {}
    110    columns: t.Dict[str, ColumnConfig] = {}
    111
    112    @validator("pre_hook", "post_hook", pre=True)
    113    def _validate_hooks(cls, v: t.Union[str, t.List[t.Union[SqlStr, str]]]) -> t.List[SqlStr]:
    114        return [SqlStr(val) for val in ensure_list(v)]
    115
    116    @validator("full_refresh", pre=True)
    117    def _validate_bool(cls, v: str) -> bool:
    118        return ensure_bool(v)
    119
    120    @validator("grants", pre=True)
    121    def _validate_grants(cls, v: t.Dict[str, str]) -> t.Dict[str, t.List[str]]:
    122        return {key: ensure_list(value) for key, value in v.items()}
    123
    124    @validator("columns", pre=True)
    125    def _validate_columns(cls, v: t.Any) -> t.Dict[str, ColumnConfig]:
    126        if not isinstance(v, dict) or all(isinstance(col, ColumnConfig) for col in v.values()):
    127            return v
    128
    129        return yaml_to_columns(v)
    130
    131    _FIELD_UPDATE_STRATEGY: t.ClassVar[t.Dict[str, UpdateStrategy]] = {
    132        **GeneralConfig._FIELD_UPDATE_STRATEGY,
    133        **{
    134            "grants": UpdateStrategy.KEY_EXTEND,
    135            "path": UpdateStrategy.IMMUTABLE,
    136            "pre-hook": UpdateStrategy.EXTEND,
    137            "post-hook": UpdateStrategy.EXTEND,
    138            "columns": UpdateStrategy.KEY_EXTEND,
    139        },
    140    }
    141
    142    @property
    143    def all_sql(self) -> SqlStr:
    144        return SqlStr("")
    145
    146    @property
    147    def table_schema(self) -> str:
    148        """
    149        Get the full schema name
    150        """
    151        return "_".join(part for part in (self.target_schema, self.schema_) if part)
    152
    153    @property
    154    def table_name(self) -> str:
    155        """
    156        Get the table name
    157        """
    158        return self.alias or self.path.stem
    159
    160    @property
    161    def model_name(self) -> str:
    162        """
    163        Get the sqlmesh model name
    164
    165        Returns:
    166            The sqlmesh model name
    167        """
    168        return ".".join(
    169            part for part in (self.database, self.table_schema, self.table_name) if part
    170        )
    171
    172    @property
    173    def model_materialization(self) -> Materialization:
    174        return Materialization.TABLE
    175
    176    @property
    177    def relation_info(self) -> AttributeDict[str, t.Any]:
    178        if self.model_materialization == Materialization.VIEW:
    179            relation_type = RelationType.View
    180        elif self.model_materialization == Materialization.EPHEMERAL:
    181            relation_type = RelationType.CTE
    182        else:
    183            relation_type = RelationType.Table
    184
    185        return AttributeDict(
    186            {
    187                "database": self.database,
    188                "schema": self.table_schema,
    189                "identifier": self.table_name,
    190                "type": relation_type.value,
    191            }
    192        )
    193
    194    def sqlmesh_model_kwargs(self, model_context: DbtContext) -> t.Dict[str, t.Any]:
    195        """Get common sqlmesh model parameters"""
    196        jinja_macros = model_context.jinja_macros.trim(self._dependencies.macros)
    197        jinja_macros.global_objs.update(
    198            {
    199                "this": self.relation_info,
    200                "schema": self.table_schema,
    201                **model_context.jinja_globals,  # type: ignore
    202            }
    203        )
    204
    205        optional_kwargs: t.Dict[str, t.Any] = {}
    206        for field in ("description", "owner", "stamp", "storage_format"):
    207            field_val = getattr(self, field, None) or self.meta.get(field, None)
    208            if field_val:
    209                optional_kwargs[field] = field_val
    210
    211        return {
    212            "columns": column_types_to_sqlmesh(self.columns) or None,
    213            "column_descriptions_": column_descriptions_to_sqlmesh(self.columns) or None,
    214            "depends_on": {model_context.refs[ref] for ref in self._dependencies.refs},
    215            "jinja_macros": jinja_macros,
    216            "path": self.path,
    217            "pre": [exp for hook in self.pre_hook for exp in d.parse(hook)],
    218            "post": [exp for hook in self.post_hook for exp in d.parse(hook)],
    219            **optional_kwargs,
    220        }
    221
    222    def render_config(self: BMC, context: DbtContext) -> BMC:
    223        rendered = super().render_config(context)
    224        rendered._dependencies = Dependencies(macros=extract_macro_references(rendered.all_sql))
    225        rendered = ModelSqlRenderer(context, rendered).enriched_config
    226
    227        rendered_dependencies = rendered._dependencies
    228        for dependency in rendered_dependencies.refs:
    229            model = context.models.get(dependency)
    230            if model and model.materialized == Materialization.EPHEMERAL:
    231                rendered._dependencies = rendered._dependencies.union(
    232                    model.render_config(context)._dependencies
    233                )
    234                rendered._dependencies.refs.discard(dependency)
    235
    236        return rendered
    237
    238    @abstractmethod
    239    def to_sqlmesh(self, context: DbtContext) -> Model:
    240        """Convert DBT model into sqlmesh Model"""
    241
    242    def _context_for_dependencies(
    243        self, context: DbtContext, dependencies: Dependencies
    244    ) -> DbtContext:
    245        model_context = context.copy()
    246
    247        model_context.sources = {
    248            name: value for name, value in context.sources.items() if name in dependencies.sources
    249        }
    250        model_context.seeds = {
    251            name: value for name, value in context.seeds.items() if name in dependencies.refs
    252        }
    253        model_context.models = {
    254            name: value for name, value in context.models.items() if name in dependencies.refs
    255        }
    256        model_context.variables = {
    257            name: value
    258            for name, value in context.variables.items()
    259            if name in dependencies.variables
    260        }
    261
    262        return model_context

###### Arguments:

- owner: The owner of the model.
- stamp: An optional arbitrary string sequence used to create new model versions without making changes to any of the functional components of the definition.
- storage_format: The storage format used to store the physical table, only applicable in certain engines. (eg. 'parquet')
- path: The file path of the model
- target_schema: The schema for the profile target
- database: Database the model is stored in
- schema: Custom schema name added to the model schema name
- alias: Relation identifier for this model instead of the filename
- pre-hook: List of SQL statements to run before the model is built
- post-hook: List of SQL statements to run after the model is built
- full_refresh: Forces the model to always do a full refresh or never do a full refresh
- grants: Set or revoke permissions to the database object for this model
- columns: Column information for the model

table_schema: str

Get the full schema name

table_name: str

Get the table name

model_name: str

Get the sqlmesh model name

###### Returns:

> The sqlmesh model name

def sqlmesh_model_kwargs(self, model_context: sqlmesh.dbt.common.DbtContext) -> Dict[str, Any]: View Source

    194    def sqlmesh_model_kwargs(self, model_context: DbtContext) -> t.Dict[str, t.Any]:
    195        """Get common sqlmesh model parameters"""
    196        jinja_macros = model_context.jinja_macros.trim(self._dependencies.macros)
    197        jinja_macros.global_objs.update(
    198            {
    199                "this": self.relation_info,
    200                "schema": self.table_schema,
    201                **model_context.jinja_globals,  # type: ignore
    202            }
    203        )
    204
    205        optional_kwargs: t.Dict[str, t.Any] = {}
    206        for field in ("description", "owner", "stamp", "storage_format"):
    207            field_val = getattr(self, field, None) or self.meta.get(field, None)
    208            if field_val:
    209                optional_kwargs[field] = field_val
    210
    211        return {
    212            "columns": column_types_to_sqlmesh(self.columns) or None,
    213            "column_descriptions_": column_descriptions_to_sqlmesh(self.columns) or None,
    214            "depends_on": {model_context.refs[ref] for ref in self._dependencies.refs},
    215            "jinja_macros": jinja_macros,
    216            "path": self.path,
    217            "pre": [exp for hook in self.pre_hook for exp in d.parse(hook)],
    218            "post": [exp for hook in self.post_hook for exp in d.parse(hook)],
    219            **optional_kwargs,
    220        }

Get common sqlmesh model parameters

def render_config(self: ~BMC, context: sqlmesh.dbt.common.DbtContext) -> ~BMC: View Source

    222    def render_config(self: BMC, context: DbtContext) -> BMC:
    223        rendered = super().render_config(context)
    224        rendered._dependencies = Dependencies(macros=extract_macro_references(rendered.all_sql))
    225        rendered = ModelSqlRenderer(context, rendered).enriched_config
    226
    227        rendered_dependencies = rendered._dependencies
    228        for dependency in rendered_dependencies.refs:
    229            model = context.models.get(dependency)
    230            if model and model.materialized == Materialization.EPHEMERAL:
    231                rendered._dependencies = rendered._dependencies.union(
    232                    model.render_config(context)._dependencies
    233                )
    234                rendered._dependencies.refs.discard(dependency)
    235
    236        return rendered

@abstractmethod
def to_sqlmesh( self, context: sqlmesh.dbt.common.DbtContext) -> Annotated[Union[sqlmesh.core.model.definition.SqlModel, sqlmesh.core.model.definition.SeedModel, sqlmesh.core.model.definition.PythonModel], FieldInfo(default=PydanticUndefined, discriminator='source_type', extra={})]: View Source

    238    @abstractmethod
    239    def to_sqlmesh(self, context: DbtContext) -> Model:
    240        """Convert DBT model into sqlmesh Model"""

Convert DBT model into sqlmesh Model

##### Inherited Members

pydantic.main.BaseModel
  BaseModel
  parse_obj
  parse_raw
  parse_file
  from_orm
  construct
  copy
  schema
  schema_json
  validate
  update_forward_refs
sqlmesh.dbt.common.GeneralConfig
  replace
sqlmesh.dbt.common.DbtConfig
  Config
sqlmesh.core.config.base.BaseConfig
  update_with
sqlmesh.utils.pydantic.PydanticModel
  dict
  json
  missing_required_fields
  extra_fields
  all_fields
  required_fields

class ModelSqlRenderer(typing.Generic[~BMC]): View Source

    265class ModelSqlRenderer(t.Generic[BMC]):
    266    def __init__(self, context: DbtContext, config: BMC):
    267        self.context = context
    268        self.config = config
    269
    270        self._captured_dependencies: Dependencies = Dependencies()
    271        self._rendered_sql: t.Optional[str] = None
    272        self._enriched_config: BMC = config.copy()
    273
    274        self._jinja_globals = create_builtin_globals(
    275            jinja_macros=context.jinja_macros,
    276            jinja_globals={
    277                **context.jinja_globals,
    278                **date_dict(c.EPOCH, c.EPOCH, c.EPOCH),
    279                "config": self._config,
    280                "ref": self._ref,
    281                "var": self._var,
    282                "source": self._source,
    283                "this": self.config.relation_info,
    284                "schema": self.config.table_schema,
    285            },
    286            engine_adapter=None,
    287        )
    288
    289        # Set the adapter separately since it requires jinja globals to passed into it.
    290        self._jinja_globals["adapter"] = ModelSqlRenderer.TrackingAdapter(
    291            self,
    292            context.jinja_macros,
    293            jinja_globals=self._jinja_globals,
    294            dialect=context.engine_adapter.dialect if context.engine_adapter else "",
    295        )
    296
    297    @property
    298    def enriched_config(self) -> BMC:
    299        if self._rendered_sql is None:
    300            self.render()
    301            self._enriched_config._dependencies = self._enriched_config._dependencies.union(
    302                self._captured_dependencies
    303            )
    304        return self._enriched_config
    305
    306    def render(self) -> str:
    307        if self._rendered_sql is None:
    308            registry = self.context.jinja_macros
    309            self._rendered_sql = (
    310                registry.build_environment(**self._jinja_globals)
    311                .from_string(self.config.all_sql)
    312                .render()
    313            )
    314        return self._rendered_sql
    315
    316    def _ref(self, package_name: str, model_name: t.Optional[str] = None) -> BaseRelation:
    317        if package_name in self.context.models:
    318            relation = BaseRelation.create(**self.context.models[package_name].relation_info)
    319        elif package_name in self.context.seeds:
    320            relation = BaseRelation.create(**self.context.seeds[package_name].relation_info)
    321        else:
    322            raise ConfigError(
    323                f"Model '{package_name}' was not found for model '{self.config.table_name}'."
    324            )
    325        self._captured_dependencies.refs.add(package_name)
    326        return relation
    327
    328    def _var(self, name: str, default: t.Optional[str] = None) -> t.Any:
    329        if default is None and name not in self.context.variables:
    330            raise ConfigError(
    331                f"Variable '{name}' was not found for model '{self.config.table_name}'."
    332            )
    333        self._captured_dependencies.variables.add(name)
    334        return self.context.variables.get(name, default)
    335
    336    def _source(self, source_name: str, table_name: str) -> BaseRelation:
    337        full_name = ".".join([source_name, table_name])
    338        if full_name not in self.context.sources:
    339            raise ConfigError(
    340                f"Source '{full_name}' was not found for model '{self.config.table_name}'."
    341            )
    342        self._captured_dependencies.sources.add(full_name)
    343        return BaseRelation.create(**self.context.sources[full_name].relation_info)
    344
    345    def _config(self, *args: t.Any, **kwargs: t.Any) -> str:
    346        if args and isinstance(args[0], dict):
    347            self._enriched_config = self._enriched_config.update_with(args[0])
    348        if kwargs:
    349            self._enriched_config = self._enriched_config.update_with(kwargs)
    350        return ""
    351
    352    class TrackingAdapter(ParsetimeAdapter):
    353        def __init__(self, outer_self: ModelSqlRenderer, *args: t.Any, **kwargs: t.Any):
    354            super().__init__(*args, **kwargs)
    355            self.outer_self = outer_self
    356            self.context = outer_self.context
    357
    358        def dispatch(self, name: str, package: t.Optional[str] = None) -> t.Callable:
    359            macros = (
    360                self.context.jinja_macros.packages.get(package, {})
    361                if package is not None
    362                else self.context.jinja_macros.root_macros
    363            )
    364            for target_name in macros:
    365                if target_name.endswith(f"__{name}"):
    366                    self.outer_self._captured_dependencies.macros.add(
    367                        MacroReference(package=package, name=target_name)
    368                    )
    369            return super().dispatch(name, package=package)

Abstract base class for generic types.

A generic type is typically declared by inheriting from this class parameterized with one or more type variables. For example, a generic mapping type might be defined as::

class Mapping(Generic[KT, VT]): def __getitem__(self, key: KT) -> VT: ... # Etc.

This class can then be used as follows::

def lookup_name(mapping: Mapping[KT, VT], key: KT, default: VT) -> VT: try: return mapping[key] except KeyError: return default

ModelSqlRenderer(context: sqlmesh.dbt.common.DbtContext, config: ~BMC) View Source

    266    def __init__(self, context: DbtContext, config: BMC):
    267        self.context = context
    268        self.config = config
    269
    270        self._captured_dependencies: Dependencies = Dependencies()
    271        self._rendered_sql: t.Optional[str] = None
    272        self._enriched_config: BMC = config.copy()
    273
    274        self._jinja_globals = create_builtin_globals(
    275            jinja_macros=context.jinja_macros,
    276            jinja_globals={
    277                **context.jinja_globals,
    278                **date_dict(c.EPOCH, c.EPOCH, c.EPOCH),
    279                "config": self._config,
    280                "ref": self._ref,
    281                "var": self._var,
    282                "source": self._source,
    283                "this": self.config.relation_info,
    284                "schema": self.config.table_schema,
    285            },
    286            engine_adapter=None,
    287        )
    288
    289        # Set the adapter separately since it requires jinja globals to passed into it.
    290        self._jinja_globals["adapter"] = ModelSqlRenderer.TrackingAdapter(
    291            self,
    292            context.jinja_macros,
    293            jinja_globals=self._jinja_globals,
    294            dialect=context.engine_adapter.dialect if context.engine_adapter else "",
    295        )

def render(self) -> str: View Source

    306    def render(self) -> str:
    307        if self._rendered_sql is None:
    308            registry = self.context.jinja_macros
    309            self._rendered_sql = (
    310                registry.build_environment(**self._jinja_globals)
    311                .from_string(self.config.all_sql)
    312                .render()
    313            )
    314        return self._rendered_sql

class ModelSqlRenderer.TrackingAdapter(sqlmesh.dbt.adapter.ParsetimeAdapter): View Source

    352    class TrackingAdapter(ParsetimeAdapter):
    353        def __init__(self, outer_self: ModelSqlRenderer, *args: t.Any, **kwargs: t.Any):
    354            super().__init__(*args, **kwargs)
    355            self.outer_self = outer_self
    356            self.context = outer_self.context
    357
    358        def dispatch(self, name: str, package: t.Optional[str] = None) -> t.Callable:
    359            macros = (
    360                self.context.jinja_macros.packages.get(package, {})
    361                if package is not None
    362                else self.context.jinja_macros.root_macros
    363            )
    364            for target_name in macros:
    365                if target_name.endswith(f"__{name}"):
    366                    self.outer_self._captured_dependencies.macros.add(
    367                        MacroReference(package=package, name=target_name)
    368                    )
    369            return super().dispatch(name, package=package)

Helper class that provides a standard way to create an ABC using inheritance.

ModelSqlRenderer.TrackingAdapter( outer_self: sqlmesh.dbt.basemodel.ModelSqlRenderer, *args: Any, **kwargs: Any) View Source

    353        def __init__(self, outer_self: ModelSqlRenderer, *args: t.Any, **kwargs: t.Any):
    354            super().__init__(*args, **kwargs)
    355            self.outer_self = outer_self
    356            self.context = outer_self.context

def dispatch(self, name: str, package: Optional[str] = None) -> Callable: View Source

    358        def dispatch(self, name: str, package: t.Optional[str] = None) -> t.Callable:
    359            macros = (
    360                self.context.jinja_macros.packages.get(package, {})
    361                if package is not None
    362                else self.context.jinja_macros.root_macros
    363            )
    364            for target_name in macros:
    365                if target_name.endswith(f"__{name}"):
    366                    self.outer_self._captured_dependencies.macros.add(
    367                        MacroReference(package=package, name=target_name)
    368                    )
    369            return super().dispatch(name, package=package)

Returns a dialect-specific version of a macro with the given name.

##### Inherited Members

sqlmesh.dbt.adapter.ParsetimeAdapter
  get_relation
  list_relations
  list_relations_without_caching
  get_columns_in_relation
  get_missing_columns
  create_schema
  drop_schema
  drop_relation
  execute
  quote