sqlmesh.dbt.project API documentation

Edit on GitHub

# sqlmesh.dbt.project

View Source

      1from __future__ import annotations
      2
      3import typing as t
      4from pathlib import Path
      5
      6from sqlmesh.dbt.common import PROJECT_FILENAME, DbtContext, load_yaml
      7from sqlmesh.dbt.package import Package, PackageLoader, ProjectConfig
      8from sqlmesh.dbt.profile import Profile
      9from sqlmesh.utils.errors import ConfigError
     10
     11
     12class Project:
     13    """Configuration for a DBT project"""
     14
     15    def __init__(
     16        self,
     17        context: DbtContext,
     18        profile: Profile,
     19        packages: t.Dict[str, Package],
     20    ):
     21        """
     22        Args:
     23            context: DBT context for the project
     24            profile: The profile associated with the project
     25            packages: The packages in this project. The project should be included
     26                      with the project name as the key
     27        """
     28        self.context = context
     29        self.profile = profile
     30        self.packages = packages
     31
     32    @classmethod
     33    def load(cls, context: DbtContext) -> Project:
     34        """
     35        Loads the configuration for the specified DBT project
     36
     37        Args:
     38            context: DBT context for this project
     39
     40        Returns:
     41            Project instance for the specified DBT project
     42        """
     43        context = context.copy()
     44
     45        project_file_path = Path(context.project_root, PROJECT_FILENAME)
     46        if not project_file_path.exists():
     47            raise ConfigError(f"Could not find {PROJECT_FILENAME} in {context.project_root}")
     48        project_yaml = load_yaml(project_file_path)
     49
     50        variables = project_yaml.get("vars", {})
     51        context.variables = {
     52            name: var for name, var in variables.items() if not isinstance(var, t.Dict)
     53        }
     54
     55        context.project_name = context.render(project_yaml.get("name", ""))
     56        if not context.project_name:
     57            raise ConfigError(f"{project_file_path.stem} must include project name.")
     58
     59        context.profile_name = (
     60            context.render(project_yaml.get("profile", "")) or context.project_name
     61        )
     62
     63        profile = Profile.load(context)
     64        context.target = (
     65            profile.targets[context.target_name]
     66            if context.target_name
     67            else profile.targets[profile.default_target]
     68        )
     69
     70        packages = {}
     71        root_loader = PackageLoader(context, ProjectConfig())
     72
     73        packages[context.project_name] = root_loader.load()
     74        project_config = root_loader.project_config
     75
     76        packages_dir = Path(
     77            context.render(project_yaml.get("packages-install-path", "dbt_packages"))
     78        )
     79        if not packages_dir.is_absolute():
     80            packages_dir = Path(context.project_root, packages_dir)
     81
     82        for path in packages_dir.glob(f"*/{PROJECT_FILENAME}"):
     83            name = context.render(load_yaml(path).get("name", ""))
     84            if not name:
     85                raise ConfigError(f"{path} must include package name")
     86
     87            package_context = context.copy()
     88            package_context.project_root = path.parent
     89            package_context.variables = {}
     90            packages[name] = PackageLoader(
     91                package_context, cls._overrides_for_package(name, project_config)
     92            ).load()
     93
     94        for name, package in packages.items():
     95            package_vars = variables.get(name)
     96
     97            if isinstance(package_vars, dict):
     98                package.variables.update(package_vars)
     99
    100        return Project(context, profile, packages)
    101
    102    @classmethod
    103    def _overrides_for_package(cls, name: str, config: ProjectConfig) -> ProjectConfig:
    104        overrides = ProjectConfig()
    105
    106        source_overrides = {
    107            scope[1:]: value
    108            for scope, value in config.source_config.items()
    109            if scope and scope[0] == name
    110        }
    111        if source_overrides:
    112            overrides.source_config = source_overrides
    113
    114        seed_overrides = {
    115            scope[1:]: value
    116            for scope, value in config.seed_config.items()
    117            if scope and scope[0] == name
    118        }
    119        if seed_overrides:
    120            overrides.seed_config = seed_overrides
    121
    122        model_overrides = {
    123            scope[1:]: value
    124            for scope, value in config.model_config.items()
    125            if scope and scope[0] == name
    126        }
    127        if model_overrides:
    128            overrides.model_config = model_overrides
    129
    130        return overrides
    131
    132    @property
    133    def project_files(self) -> t.Set[Path]:
    134        paths = {self.profile.path}
    135        for package in self.packages.values():
    136            paths.update(package.files)
    137
    138        return paths

class Project: View Source

     13class Project:
     14    """Configuration for a DBT project"""
     15
     16    def __init__(
     17        self,
     18        context: DbtContext,
     19        profile: Profile,
     20        packages: t.Dict[str, Package],
     21    ):
     22        """
     23        Args:
     24            context: DBT context for the project
     25            profile: The profile associated with the project
     26            packages: The packages in this project. The project should be included
     27                      with the project name as the key
     28        """
     29        self.context = context
     30        self.profile = profile
     31        self.packages = packages
     32
     33    @classmethod
     34    def load(cls, context: DbtContext) -> Project:
     35        """
     36        Loads the configuration for the specified DBT project
     37
     38        Args:
     39            context: DBT context for this project
     40
     41        Returns:
     42            Project instance for the specified DBT project
     43        """
     44        context = context.copy()
     45
     46        project_file_path = Path(context.project_root, PROJECT_FILENAME)
     47        if not project_file_path.exists():
     48            raise ConfigError(f"Could not find {PROJECT_FILENAME} in {context.project_root}")
     49        project_yaml = load_yaml(project_file_path)
     50
     51        variables = project_yaml.get("vars", {})
     52        context.variables = {
     53            name: var for name, var in variables.items() if not isinstance(var, t.Dict)
     54        }
     55
     56        context.project_name = context.render(project_yaml.get("name", ""))
     57        if not context.project_name:
     58            raise ConfigError(f"{project_file_path.stem} must include project name.")
     59
     60        context.profile_name = (
     61            context.render(project_yaml.get("profile", "")) or context.project_name
     62        )
     63
     64        profile = Profile.load(context)
     65        context.target = (
     66            profile.targets[context.target_name]
     67            if context.target_name
     68            else profile.targets[profile.default_target]
     69        )
     70
     71        packages = {}
     72        root_loader = PackageLoader(context, ProjectConfig())
     73
     74        packages[context.project_name] = root_loader.load()
     75        project_config = root_loader.project_config
     76
     77        packages_dir = Path(
     78            context.render(project_yaml.get("packages-install-path", "dbt_packages"))
     79        )
     80        if not packages_dir.is_absolute():
     81            packages_dir = Path(context.project_root, packages_dir)
     82
     83        for path in packages_dir.glob(f"*/{PROJECT_FILENAME}"):
     84            name = context.render(load_yaml(path).get("name", ""))
     85            if not name:
     86                raise ConfigError(f"{path} must include package name")
     87
     88            package_context = context.copy()
     89            package_context.project_root = path.parent
     90            package_context.variables = {}
     91            packages[name] = PackageLoader(
     92                package_context, cls._overrides_for_package(name, project_config)
     93            ).load()
     94
     95        for name, package in packages.items():
     96            package_vars = variables.get(name)
     97
     98            if isinstance(package_vars, dict):
     99                package.variables.update(package_vars)
    100
    101        return Project(context, profile, packages)
    102
    103    @classmethod
    104    def _overrides_for_package(cls, name: str, config: ProjectConfig) -> ProjectConfig:
    105        overrides = ProjectConfig()
    106
    107        source_overrides = {
    108            scope[1:]: value
    109            for scope, value in config.source_config.items()
    110            if scope and scope[0] == name
    111        }
    112        if source_overrides:
    113            overrides.source_config = source_overrides
    114
    115        seed_overrides = {
    116            scope[1:]: value
    117            for scope, value in config.seed_config.items()
    118            if scope and scope[0] == name
    119        }
    120        if seed_overrides:
    121            overrides.seed_config = seed_overrides
    122
    123        model_overrides = {
    124            scope[1:]: value
    125            for scope, value in config.model_config.items()
    126            if scope and scope[0] == name
    127        }
    128        if model_overrides:
    129            overrides.model_config = model_overrides
    130
    131        return overrides
    132
    133    @property
    134    def project_files(self) -> t.Set[Path]:
    135        paths = {self.profile.path}
    136        for package in self.packages.values():
    137            paths.update(package.files)
    138
    139        return paths

Configuration for a DBT project

Project( context: sqlmesh.dbt.common.DbtContext, profile: sqlmesh.dbt.profile.Profile, packages: Dict[str, sqlmesh.dbt.package.Package]) View Source

    16    def __init__(
    17        self,
    18        context: DbtContext,
    19        profile: Profile,
    20        packages: t.Dict[str, Package],
    21    ):
    22        """
    23        Args:
    24            context: DBT context for the project
    25            profile: The profile associated with the project
    26            packages: The packages in this project. The project should be included
    27                      with the project name as the key
    28        """
    29        self.context = context
    30        self.profile = profile
    31        self.packages = packages

###### Arguments:

- context: DBT context for the project
- profile: The profile associated with the project
- packages: The packages in this project. The project should be included with the project name as the key

@classmethod
def load( cls, context: sqlmesh.dbt.common.DbtContext) -> sqlmesh.dbt.project.Project: View Source

     33    @classmethod
     34    def load(cls, context: DbtContext) -> Project:
     35        """
     36        Loads the configuration for the specified DBT project
     37
     38        Args:
     39            context: DBT context for this project
     40
     41        Returns:
     42            Project instance for the specified DBT project
     43        """
     44        context = context.copy()
     45
     46        project_file_path = Path(context.project_root, PROJECT_FILENAME)
     47        if not project_file_path.exists():
     48            raise ConfigError(f"Could not find {PROJECT_FILENAME} in {context.project_root}")
     49        project_yaml = load_yaml(project_file_path)
     50
     51        variables = project_yaml.get("vars", {})
     52        context.variables = {
     53            name: var for name, var in variables.items() if not isinstance(var, t.Dict)
     54        }
     55
     56        context.project_name = context.render(project_yaml.get("name", ""))
     57        if not context.project_name:
     58            raise ConfigError(f"{project_file_path.stem} must include project name.")
     59
     60        context.profile_name = (
     61            context.render(project_yaml.get("profile", "")) or context.project_name
     62        )
     63
     64        profile = Profile.load(context)
     65        context.target = (
     66            profile.targets[context.target_name]
     67            if context.target_name
     68            else profile.targets[profile.default_target]
     69        )
     70
     71        packages = {}
     72        root_loader = PackageLoader(context, ProjectConfig())
     73
     74        packages[context.project_name] = root_loader.load()
     75        project_config = root_loader.project_config
     76
     77        packages_dir = Path(
     78            context.render(project_yaml.get("packages-install-path", "dbt_packages"))
     79        )
     80        if not packages_dir.is_absolute():
     81            packages_dir = Path(context.project_root, packages_dir)
     82
     83        for path in packages_dir.glob(f"*/{PROJECT_FILENAME}"):
     84            name = context.render(load_yaml(path).get("name", ""))
     85            if not name:
     86                raise ConfigError(f"{path} must include package name")
     87
     88            package_context = context.copy()
     89            package_context.project_root = path.parent
     90            package_context.variables = {}
     91            packages[name] = PackageLoader(
     92                package_context, cls._overrides_for_package(name, project_config)
     93            ).load()
     94
     95        for name, package in packages.items():
     96            package_vars = variables.get(name)
     97
     98            if isinstance(package_vars, dict):
     99                package.variables.update(package_vars)
    100
    101        return Project(context, profile, packages)

Loads the configuration for the specified DBT project

###### Arguments:

- context: DBT context for this project

###### Returns:

> Project instance for the specified DBT project