sqlmesh.dbt.package API documentation

Edit on GitHub

# sqlmesh.dbt.package

View Source

      1from __future__ import annotations
      2
      3import typing as t
      4from pathlib import Path
      5
      6from sqlmesh.dbt.common import (
      7    PROJECT_FILENAME,
      8    BaseConfig,
      9    DbtContext,
     10    SqlStr,
     11    load_yaml,
     12)
     13from sqlmesh.dbt.model import ModelConfig
     14from sqlmesh.dbt.seed import SeedConfig
     15from sqlmesh.dbt.source import SourceConfig
     16from sqlmesh.utils.errors import ConfigError
     17from sqlmesh.utils.jinja import MacroExtractor, MacroInfo
     18from sqlmesh.utils.pydantic import PydanticModel
     19
     20if t.TYPE_CHECKING:
     21    C = t.TypeVar("C", bound=BaseConfig)
     22
     23Scope = t.Union[t.Tuple[()], t.Tuple[str, ...]]
     24ScopedSources = t.Dict[Scope, SourceConfig]
     25ScopedSeeds = t.Dict[Scope, SeedConfig]
     26ScopedModels = t.Dict[Scope, ModelConfig]
     27
     28
     29class ProjectConfig(PydanticModel):
     30    """Class to contain configuration from dbt_project.yml"""
     31
     32    source_config: ScopedSources = {(): SourceConfig()}
     33    seed_config: ScopedSeeds = {(): SeedConfig()}
     34    model_config: ScopedModels = {(): ModelConfig()}
     35
     36
     37class Package(PydanticModel):
     38    """Class to contain package configuration"""
     39
     40    sources: t.Dict[str, SourceConfig]
     41    seeds: t.Dict[str, SeedConfig]
     42    models: t.Dict[str, ModelConfig]
     43    variables: t.Dict[str, t.Any]
     44    macros: t.Dict[str, MacroInfo]
     45    files: t.Set[Path]
     46
     47
     48class PackageLoader:
     49    """Loader for DBT packages"""
     50
     51    def __init__(self, context: DbtContext, overrides: ProjectConfig):
     52        self._context = context.copy()
     53        self._overrides = overrides
     54        self._config_paths: t.Set[Path] = set()
     55        self.project_config = ProjectConfig()
     56
     57    def load(self) -> Package:
     58        """
     59        Loads the specified package.
     60
     61        Returns:
     62            Package containing the configuration found within this package
     63        """
     64        project_file_path = Path(self._context.project_root, PROJECT_FILENAME)
     65        if not project_file_path.exists():
     66            raise ConfigError(f"Could not find {PROJECT_FILENAME} in {self._context.project_root}")
     67
     68        self._config_paths.add(project_file_path)
     69        project_yaml = load_yaml(project_file_path)
     70
     71        self._package_name = self._context.render(project_yaml.get("name", ""))
     72
     73        # Only include globally-scoped variables (i.e. filter out the package-scoped ones)
     74        self._context.variables = {
     75            var: value
     76            for var, value in project_yaml.get("vars", {}).items()
     77            if not isinstance(value, dict)
     78        }
     79
     80        self._load_project_config(project_yaml)
     81
     82        models_dirs = [
     83            Path(self._context.project_root, self._context.render(dir))
     84            for dir in project_yaml.get("model-paths") or ["models"]
     85        ]
     86        models, sources = self._load_models(models_dirs)
     87
     88        seeds_dirs = [
     89            Path(self._context.project_root, self._context.render(dir))
     90            for dir in project_yaml.get("seed-paths") or ["seeds"]
     91        ]
     92        seeds = self._load_seeds(seeds_dirs)
     93
     94        macros_dirs = [
     95            Path(self._context.project_root, self._context.render(dir))
     96            for dir in project_yaml.get("macro-paths") or ["macros"]
     97        ]
     98        macros = self._load_macros(macros_dirs)
     99
    100        return Package(
    101            models=models,
    102            sources=sources,
    103            seeds=seeds,
    104            variables=self._context.variables,
    105            macros=macros,
    106            files=self._config_paths,
    107        )
    108
    109    def _load_project_config(self, yaml: t.Dict[str, t.Any]) -> None:
    110        def load_config(
    111            data: t.Dict[str, t.Any],
    112            parent: C,
    113            scope: Scope,
    114            scoped_configs: t.Optional[t.Dict[Scope, C]] = None,
    115        ) -> t.Dict[Scope, C]:
    116            """Recursively loads config and nested configs in the provided yaml"""
    117            scoped_configs = scoped_configs or {}
    118
    119            config_fields = parent.all_fields()
    120
    121            nested_config = {}
    122            fields = {}
    123            for key, value in data.items():
    124                if key.startswith("+") or key in config_fields or not isinstance(value, dict):
    125                    fields[key[1:]] = value
    126                else:
    127                    nested_config[key] = value
    128
    129            config = parent.update_with(fields)
    130            scoped_configs[scope] = config
    131            for key, value in nested_config.items():
    132                nested_scope = (*scope, key)
    133                load_config(value, config, nested_scope, scoped_configs)
    134
    135            return scoped_configs
    136
    137        scope = ()
    138        self.project_config.source_config = load_config(
    139            yaml.get("sources", {}), SourceConfig(), scope
    140        )
    141        self.project_config.seed_config = load_config(
    142            yaml.get("seeds", {}),
    143            SeedConfig(target_schema=self._context.target.schema_),
    144            scope,
    145        )
    146        self.project_config.model_config = load_config(
    147            yaml.get("models", {}),
    148            ModelConfig(target_schema=self._context.target.schema_),
    149            scope,
    150        )
    151
    152    def _load_models(
    153        self, models_dirs: t.List[Path]
    154    ) -> t.Tuple[t.Dict[str, ModelConfig], t.Dict[str, SourceConfig]]:
    155        """
    156        Loads the configuration of all models within the DBT project.
    157
    158        Args:
    159            models_dirs: List of dirs containing models
    160
    161        Returns:
    162            Tuple containing Dicts of model configs, source configs, and list of config files
    163            used by them
    164        """
    165        models: t.Dict[str, ModelConfig] = {}
    166        sources: t.Dict[str, SourceConfig] = {}
    167
    168        for root in models_dirs:
    169            # Layer on configs in properties files
    170            for path in root.glob("**/*.yml"):
    171                self._config_paths.add(path)
    172
    173                scope = self._scope_from_path(path, root)
    174                properties_yaml = load_yaml(path)
    175
    176                self._load_config_section_from_properties(
    177                    properties_yaml, "models", scope, self.project_config.model_config
    178                )
    179
    180                source_configs_in_file = self._load_sources_config_from_properties(
    181                    properties_yaml, scope, self.project_config.source_config
    182                )
    183                sources.update(source_configs_in_file)
    184
    185            # Layer on configs from the model file and create model configs
    186            for path in root.glob("**/*.sql"):
    187                self._config_paths.add(path)
    188
    189                scope = self._scope_from_path(path, root)
    190                model_config = self._load_model_config_from_model(path, scope)
    191                model_config.update_with(self._overridden_model_fields(scope))
    192                if model_config.enabled and model_config.table_name:
    193                    models[model_config.table_name] = model_config
    194
    195        return (models, sources)
    196
    197    def _load_seeds(
    198        self,
    199        seeds_dirs: t.List[Path],
    200    ) -> t.Dict[str, SeedConfig]:
    201        """
    202        Loads the configuration of all seeds within the DBT project.
    203
    204        Args:
    205            seeds_dirs: List of dirs containing seeds
    206
    207        Returns:
    208            Tuple containing Dicts of seed configs and list of config files used by them
    209        """
    210        seeds: t.Dict[str, SeedConfig] = {}
    211
    212        for root in seeds_dirs:
    213            # Layer on configs in properties files
    214            for path in root.glob("**/*.yml"):
    215                self._config_paths.add(path)
    216
    217                scope = self._scope_from_path(path, root)
    218                properties_yaml = load_yaml(path)
    219
    220                self._load_config_section_from_properties(
    221                    properties_yaml, "seeds", scope, self.project_config.seed_config
    222                )
    223
    224            # Layer on configs from the model file and create seed configs
    225            for path in root.glob("**/*.csv"):
    226                self._config_paths.add(path)
    227
    228                scope = self._scope_from_path(path, root)
    229                seed_config = self._config_for_scope(scope, self.project_config.seed_config).copy()
    230                seed_config.update_with(self._overridden_seed_fields(scope))
    231                seed_config.path = path
    232                if seed_config.enabled:
    233                    seeds[path.stem] = seed_config
    234
    235        return seeds
    236
    237    def _load_macros(self, macros_dirs: t.List[Path]) -> t.Dict[str, MacroInfo]:
    238        macros: t.Dict[str, MacroInfo] = {}
    239
    240        for root in macros_dirs:
    241            for path in root.glob("**/*.sql"):
    242                macros.update(self._load_macro_file(path))
    243
    244        return macros
    245
    246    def _load_macro_file(self, path: Path) -> t.Dict[str, MacroInfo]:
    247        self._config_paths.add(path)
    248        with open(path, mode="r", encoding="utf8") as file:
    249            return MacroExtractor().extract(file.read())
    250
    251    def _load_config_section_from_properties(
    252        self,
    253        yaml: t.Dict[str, t.Any],
    254        section_name: str,
    255        scope: Scope,
    256        scoped_configs: t.Dict[Scope, C],
    257    ) -> None:
    258        section_yaml = yaml.get(section_name)
    259        if not section_yaml:
    260            return
    261
    262        for value in section_yaml:
    263            fields = value.get("config")
    264            if not fields:
    265                continue
    266
    267            model_scope = (*scope, value["name"])
    268            scoped_configs[model_scope] = self._config_for_scope(scope, scoped_configs).update_with(
    269                fields
    270            )
    271
    272    def _load_sources_config_from_properties(
    273        self,
    274        properties_yaml: t.Dict[str, t.Any],
    275        scope: Scope,
    276        scoped_configs: t.Dict[Scope, SourceConfig],
    277    ) -> t.Dict[str, SourceConfig]:
    278        sources_yaml = properties_yaml.get("sources")
    279        if not sources_yaml:
    280            return {}
    281
    282        configs = {}
    283
    284        for source in sources_yaml:
    285            source_name = source["name"]
    286            source_config = self._config_for_scope((*scope, source_name), scoped_configs)
    287
    288            source_config.schema_ = source_name
    289            config_fields = source.get("config")
    290            if config_fields:
    291                source_config = source_config.update_with(config_fields)
    292
    293            for table in source["tables"]:
    294                table_name = table["name"]
    295                table_config = source_config.copy()
    296
    297                table_config.identifier = table_name
    298                config_fields = source.get("config")
    299                if config_fields:
    300                    table_config = table_config.update_with(config_fields)
    301
    302                table_config.config_name = f"{source_name}.{table_name}"
    303                if table_config.enabled:
    304                    configs[table_config.config_name] = table_config
    305
    306        return configs
    307
    308    def _load_model_config_from_model(self, filepath: Path, scope: Scope) -> ModelConfig:
    309        with filepath.open(encoding="utf-8") as file:
    310            sql = file.read()
    311
    312        model_config = self._config_for_scope(scope, self.project_config.model_config).copy(
    313            update={"path": filepath}
    314        )
    315        model_config.sql = SqlStr(sql)
    316
    317        return model_config
    318
    319    def _overridden_source_fields(self, scope: Scope) -> t.Dict[str, t.Any]:
    320        return self._overridden_fields(scope, self._overrides.source_config)
    321
    322    def _overridden_model_fields(self, scope: Scope) -> t.Dict[str, t.Any]:
    323        return self._overridden_fields(scope, self._overrides.model_config)
    324
    325    def _overridden_seed_fields(self, scope: Scope) -> t.Dict[str, t.Any]:
    326        return self._overridden_fields(scope, self._overrides.seed_config)
    327
    328    def _overridden_fields(self, scope: Scope, config: t.Dict[Scope, C]) -> t.Dict[str, t.Any]:
    329        return self._config_for_scope(scope, config).dict(exclude_defaults=True, exclude_none=True)
    330
    331    def _scope_from_path(self, path: Path, root_path: Path) -> Scope:
    332        """
    333        DBT rolls-up configuration based o the directory structure. Scope mimics this structure,
    334        building a tuple containing the project name and directories from project root to the file,
    335        omitting the "models" directory and filename if a properties file.
    336        """
    337        path_from_root = path.relative_to(root_path)
    338        scope = (self._package_name, *path_from_root.parts[:-1])
    339        if path.match("*.sql") or path.match("*.csv"):
    340            scope = (*scope, path_from_root.stem)
    341        return scope
    342
    343    def _config_for_scope(self, scope: Scope, configs: t.Dict[Scope, C]) -> C:
    344        return configs.get(scope) or self._config_for_scope(scope[0:-1], configs)

class ProjectConfig(sqlmesh.utils.pydantic.PydanticModel): View Source

    30class ProjectConfig(PydanticModel):
    31    """Class to contain configuration from dbt_project.yml"""
    32
    33    source_config: ScopedSources = {(): SourceConfig()}
    34    seed_config: ScopedSeeds = {(): SeedConfig()}
    35    model_config: ScopedModels = {(): ModelConfig()}

Class to contain configuration from dbt_project.yml

##### Inherited Members

pydantic.main.BaseModel
  BaseModel
  parse_obj
  parse_raw
  parse_file
  from_orm
  construct
  copy
  schema
  schema_json
  validate
  update_forward_refs
sqlmesh.utils.pydantic.PydanticModel
  Config
  dict
  json
  missing_required_fields
  extra_fields
  all_fields
  required_fields

class Package(sqlmesh.utils.pydantic.PydanticModel): View Source

    38class Package(PydanticModel):
    39    """Class to contain package configuration"""
    40
    41    sources: t.Dict[str, SourceConfig]
    42    seeds: t.Dict[str, SeedConfig]
    43    models: t.Dict[str, ModelConfig]
    44    variables: t.Dict[str, t.Any]
    45    macros: t.Dict[str, MacroInfo]
    46    files: t.Set[Path]

Class to contain package configuration

##### Inherited Members

pydantic.main.BaseModel
  BaseModel
  parse_obj
  parse_raw
  parse_file
  from_orm
  construct
  copy
  schema
  schema_json
  validate
  update_forward_refs
sqlmesh.utils.pydantic.PydanticModel
  Config
  dict
  json
  missing_required_fields
  extra_fields
  all_fields
  required_fields

class PackageLoader: View Source

     49class PackageLoader:
     50    """Loader for DBT packages"""
     51
     52    def __init__(self, context: DbtContext, overrides: ProjectConfig):
     53        self._context = context.copy()
     54        self._overrides = overrides
     55        self._config_paths: t.Set[Path] = set()
     56        self.project_config = ProjectConfig()
     57
     58    def load(self) -> Package:
     59        """
     60        Loads the specified package.
     61
     62        Returns:
     63            Package containing the configuration found within this package
     64        """
     65        project_file_path = Path(self._context.project_root, PROJECT_FILENAME)
     66        if not project_file_path.exists():
     67            raise ConfigError(f"Could not find {PROJECT_FILENAME} in {self._context.project_root}")
     68
     69        self._config_paths.add(project_file_path)
     70        project_yaml = load_yaml(project_file_path)
     71
     72        self._package_name = self._context.render(project_yaml.get("name", ""))
     73
     74        # Only include globally-scoped variables (i.e. filter out the package-scoped ones)
     75        self._context.variables = {
     76            var: value
     77            for var, value in project_yaml.get("vars", {}).items()
     78            if not isinstance(value, dict)
     79        }
     80
     81        self._load_project_config(project_yaml)
     82
     83        models_dirs = [
     84            Path(self._context.project_root, self._context.render(dir))
     85            for dir in project_yaml.get("model-paths") or ["models"]
     86        ]
     87        models, sources = self._load_models(models_dirs)
     88
     89        seeds_dirs = [
     90            Path(self._context.project_root, self._context.render(dir))
     91            for dir in project_yaml.get("seed-paths") or ["seeds"]
     92        ]
     93        seeds = self._load_seeds(seeds_dirs)
     94
     95        macros_dirs = [
     96            Path(self._context.project_root, self._context.render(dir))
     97            for dir in project_yaml.get("macro-paths") or ["macros"]
     98        ]
     99        macros = self._load_macros(macros_dirs)
    100
    101        return Package(
    102            models=models,
    103            sources=sources,
    104            seeds=seeds,
    105            variables=self._context.variables,
    106            macros=macros,
    107            files=self._config_paths,
    108        )
    109
    110    def _load_project_config(self, yaml: t.Dict[str, t.Any]) -> None:
    111        def load_config(
    112            data: t.Dict[str, t.Any],
    113            parent: C,
    114            scope: Scope,
    115            scoped_configs: t.Optional[t.Dict[Scope, C]] = None,
    116        ) -> t.Dict[Scope, C]:
    117            """Recursively loads config and nested configs in the provided yaml"""
    118            scoped_configs = scoped_configs or {}
    119
    120            config_fields = parent.all_fields()
    121
    122            nested_config = {}
    123            fields = {}
    124            for key, value in data.items():
    125                if key.startswith("+") or key in config_fields or not isinstance(value, dict):
    126                    fields[key[1:]] = value
    127                else:
    128                    nested_config[key] = value
    129
    130            config = parent.update_with(fields)
    131            scoped_configs[scope] = config
    132            for key, value in nested_config.items():
    133                nested_scope = (*scope, key)
    134                load_config(value, config, nested_scope, scoped_configs)
    135
    136            return scoped_configs
    137
    138        scope = ()
    139        self.project_config.source_config = load_config(
    140            yaml.get("sources", {}), SourceConfig(), scope
    141        )
    142        self.project_config.seed_config = load_config(
    143            yaml.get("seeds", {}),
    144            SeedConfig(target_schema=self._context.target.schema_),
    145            scope,
    146        )
    147        self.project_config.model_config = load_config(
    148            yaml.get("models", {}),
    149            ModelConfig(target_schema=self._context.target.schema_),
    150            scope,
    151        )
    152
    153    def _load_models(
    154        self, models_dirs: t.List[Path]
    155    ) -> t.Tuple[t.Dict[str, ModelConfig], t.Dict[str, SourceConfig]]:
    156        """
    157        Loads the configuration of all models within the DBT project.
    158
    159        Args:
    160            models_dirs: List of dirs containing models
    161
    162        Returns:
    163            Tuple containing Dicts of model configs, source configs, and list of config files
    164            used by them
    165        """
    166        models: t.Dict[str, ModelConfig] = {}
    167        sources: t.Dict[str, SourceConfig] = {}
    168
    169        for root in models_dirs:
    170            # Layer on configs in properties files
    171            for path in root.glob("**/*.yml"):
    172                self._config_paths.add(path)
    173
    174                scope = self._scope_from_path(path, root)
    175                properties_yaml = load_yaml(path)
    176
    177                self._load_config_section_from_properties(
    178                    properties_yaml, "models", scope, self.project_config.model_config
    179                )
    180
    181                source_configs_in_file = self._load_sources_config_from_properties(
    182                    properties_yaml, scope, self.project_config.source_config
    183                )
    184                sources.update(source_configs_in_file)
    185
    186            # Layer on configs from the model file and create model configs
    187            for path in root.glob("**/*.sql"):
    188                self._config_paths.add(path)
    189
    190                scope = self._scope_from_path(path, root)
    191                model_config = self._load_model_config_from_model(path, scope)
    192                model_config.update_with(self._overridden_model_fields(scope))
    193                if model_config.enabled and model_config.table_name:
    194                    models[model_config.table_name] = model_config
    195
    196        return (models, sources)
    197
    198    def _load_seeds(
    199        self,
    200        seeds_dirs: t.List[Path],
    201    ) -> t.Dict[str, SeedConfig]:
    202        """
    203        Loads the configuration of all seeds within the DBT project.
    204
    205        Args:
    206            seeds_dirs: List of dirs containing seeds
    207
    208        Returns:
    209            Tuple containing Dicts of seed configs and list of config files used by them
    210        """
    211        seeds: t.Dict[str, SeedConfig] = {}
    212
    213        for root in seeds_dirs:
    214            # Layer on configs in properties files
    215            for path in root.glob("**/*.yml"):
    216                self._config_paths.add(path)
    217
    218                scope = self._scope_from_path(path, root)
    219                properties_yaml = load_yaml(path)
    220
    221                self._load_config_section_from_properties(
    222                    properties_yaml, "seeds", scope, self.project_config.seed_config
    223                )
    224
    225            # Layer on configs from the model file and create seed configs
    226            for path in root.glob("**/*.csv"):
    227                self._config_paths.add(path)
    228
    229                scope = self._scope_from_path(path, root)
    230                seed_config = self._config_for_scope(scope, self.project_config.seed_config).copy()
    231                seed_config.update_with(self._overridden_seed_fields(scope))
    232                seed_config.path = path
    233                if seed_config.enabled:
    234                    seeds[path.stem] = seed_config
    235
    236        return seeds
    237
    238    def _load_macros(self, macros_dirs: t.List[Path]) -> t.Dict[str, MacroInfo]:
    239        macros: t.Dict[str, MacroInfo] = {}
    240
    241        for root in macros_dirs:
    242            for path in root.glob("**/*.sql"):
    243                macros.update(self._load_macro_file(path))
    244
    245        return macros
    246
    247    def _load_macro_file(self, path: Path) -> t.Dict[str, MacroInfo]:
    248        self._config_paths.add(path)
    249        with open(path, mode="r", encoding="utf8") as file:
    250            return MacroExtractor().extract(file.read())
    251
    252    def _load_config_section_from_properties(
    253        self,
    254        yaml: t.Dict[str, t.Any],
    255        section_name: str,
    256        scope: Scope,
    257        scoped_configs: t.Dict[Scope, C],
    258    ) -> None:
    259        section_yaml = yaml.get(section_name)
    260        if not section_yaml:
    261            return
    262
    263        for value in section_yaml:
    264            fields = value.get("config")
    265            if not fields:
    266                continue
    267
    268            model_scope = (*scope, value["name"])
    269            scoped_configs[model_scope] = self._config_for_scope(scope, scoped_configs).update_with(
    270                fields
    271            )
    272
    273    def _load_sources_config_from_properties(
    274        self,
    275        properties_yaml: t.Dict[str, t.Any],
    276        scope: Scope,
    277        scoped_configs: t.Dict[Scope, SourceConfig],
    278    ) -> t.Dict[str, SourceConfig]:
    279        sources_yaml = properties_yaml.get("sources")
    280        if not sources_yaml:
    281            return {}
    282
    283        configs = {}
    284
    285        for source in sources_yaml:
    286            source_name = source["name"]
    287            source_config = self._config_for_scope((*scope, source_name), scoped_configs)
    288
    289            source_config.schema_ = source_name
    290            config_fields = source.get("config")
    291            if config_fields:
    292                source_config = source_config.update_with(config_fields)
    293
    294            for table in source["tables"]:
    295                table_name = table["name"]
    296                table_config = source_config.copy()
    297
    298                table_config.identifier = table_name
    299                config_fields = source.get("config")
    300                if config_fields:
    301                    table_config = table_config.update_with(config_fields)
    302
    303                table_config.config_name = f"{source_name}.{table_name}"
    304                if table_config.enabled:
    305                    configs[table_config.config_name] = table_config
    306
    307        return configs
    308
    309    def _load_model_config_from_model(self, filepath: Path, scope: Scope) -> ModelConfig:
    310        with filepath.open(encoding="utf-8") as file:
    311            sql = file.read()
    312
    313        model_config = self._config_for_scope(scope, self.project_config.model_config).copy(
    314            update={"path": filepath}
    315        )
    316        model_config.sql = SqlStr(sql)
    317
    318        return model_config
    319
    320    def _overridden_source_fields(self, scope: Scope) -> t.Dict[str, t.Any]:
    321        return self._overridden_fields(scope, self._overrides.source_config)
    322
    323    def _overridden_model_fields(self, scope: Scope) -> t.Dict[str, t.Any]:
    324        return self._overridden_fields(scope, self._overrides.model_config)
    325
    326    def _overridden_seed_fields(self, scope: Scope) -> t.Dict[str, t.Any]:
    327        return self._overridden_fields(scope, self._overrides.seed_config)
    328
    329    def _overridden_fields(self, scope: Scope, config: t.Dict[Scope, C]) -> t.Dict[str, t.Any]:
    330        return self._config_for_scope(scope, config).dict(exclude_defaults=True, exclude_none=True)
    331
    332    def _scope_from_path(self, path: Path, root_path: Path) -> Scope:
    333        """
    334        DBT rolls-up configuration based o the directory structure. Scope mimics this structure,
    335        building a tuple containing the project name and directories from project root to the file,
    336        omitting the "models" directory and filename if a properties file.
    337        """
    338        path_from_root = path.relative_to(root_path)
    339        scope = (self._package_name, *path_from_root.parts[:-1])
    340        if path.match("*.sql") or path.match("*.csv"):
    341            scope = (*scope, path_from_root.stem)
    342        return scope
    343
    344    def _config_for_scope(self, scope: Scope, configs: t.Dict[Scope, C]) -> C:
    345        return configs.get(scope) or self._config_for_scope(scope[0:-1], configs)

Loader for DBT packages

PackageLoader( context: sqlmesh.dbt.common.DbtContext, overrides: sqlmesh.dbt.package.ProjectConfig) View Source

    52    def __init__(self, context: DbtContext, overrides: ProjectConfig):
    53        self._context = context.copy()
    54        self._overrides = overrides
    55        self._config_paths: t.Set[Path] = set()
    56        self.project_config = ProjectConfig()

def load(self) -> sqlmesh.dbt.package.Package: View Source

     58    def load(self) -> Package:
     59        """
     60        Loads the specified package.
     61
     62        Returns:
     63            Package containing the configuration found within this package
     64        """
     65        project_file_path = Path(self._context.project_root, PROJECT_FILENAME)
     66        if not project_file_path.exists():
     67            raise ConfigError(f"Could not find {PROJECT_FILENAME} in {self._context.project_root}")
     68
     69        self._config_paths.add(project_file_path)
     70        project_yaml = load_yaml(project_file_path)
     71
     72        self._package_name = self._context.render(project_yaml.get("name", ""))
     73
     74        # Only include globally-scoped variables (i.e. filter out the package-scoped ones)
     75        self._context.variables = {
     76            var: value
     77            for var, value in project_yaml.get("vars", {}).items()
     78            if not isinstance(value, dict)
     79        }
     80
     81        self._load_project_config(project_yaml)
     82
     83        models_dirs = [
     84            Path(self._context.project_root, self._context.render(dir))
     85            for dir in project_yaml.get("model-paths") or ["models"]
     86        ]
     87        models, sources = self._load_models(models_dirs)
     88
     89        seeds_dirs = [
     90            Path(self._context.project_root, self._context.render(dir))
     91            for dir in project_yaml.get("seed-paths") or ["seeds"]
     92        ]
     93        seeds = self._load_seeds(seeds_dirs)
     94
     95        macros_dirs = [
     96            Path(self._context.project_root, self._context.render(dir))
     97            for dir in project_yaml.get("macro-paths") or ["macros"]
     98        ]
     99        macros = self._load_macros(macros_dirs)
    100
    101        return Package(
    102            models=models,
    103            sources=sources,
    104            seeds=seeds,
    105            variables=self._context.variables,
    106            macros=macros,
    107            files=self._config_paths,
    108        )

Loads the specified package.

###### Returns:

> Package containing the configuration found within this package