sqlmesh.dbt.profile API documentation

Edit on GitHub

# sqlmesh.dbt.profile

View Source

      1from __future__ import annotations
      2
      3import typing as t
      4from pathlib import Path
      5
      6from sqlmesh.core.config.connection import ConnectionConfig
      7from sqlmesh.dbt.common import PROJECT_FILENAME, DbtContext, load_yaml
      8from sqlmesh.dbt.target import TargetConfig
      9from sqlmesh.utils.errors import ConfigError
     10
     11
     12class Profile:
     13    """
     14    A class to read DBT profiles and obtain the project's target data warehouse configuration
     15    """
     16
     17    PROFILE_FILE = "profiles.yml"
     18
     19    def __init__(
     20        self,
     21        path: Path,
     22        targets: t.Dict[str, TargetConfig],
     23        default_target: str,
     24    ):
     25        """
     26        Args:
     27            path: Path to the profile file
     28            targets: Dict of targets defined for the project
     29            default_target: Name of the default target for the proejct
     30        """
     31        self.path = path
     32        self.targets = targets
     33        self.default_target = default_target
     34
     35    @classmethod
     36    def load(cls, context: DbtContext) -> Profile:
     37        """
     38        Loads the profile for the specified project
     39
     40        Args:
     41            context: DBT context for this profile
     42
     43        Returns:
     44            The Profile for the specified project
     45        """
     46        if not context.profile_name:
     47            project_file = Path(context.project_root, PROJECT_FILENAME)
     48            if not project_file.exists():
     49                raise ConfigError(f"Could not find {PROJECT_FILENAME} in {context.project_root}")
     50
     51            project_yaml = load_yaml(project_file)
     52            context.profile_name = context.render(
     53                project_yaml.get("profile", "")
     54            ) or context.render(project_yaml.get("name", ""))
     55            if not context.profile_name:
     56                raise ConfigError(f"{project_file.stem} must include project name.")
     57
     58        profile_filepath = cls._find_profile(context.project_root)
     59        if not profile_filepath:
     60            raise ConfigError(f"{cls.PROFILE_FILE} not found.")
     61
     62        targets, default_target = cls._read_profile(profile_filepath, context)
     63        return Profile(profile_filepath, targets, default_target)
     64
     65    @classmethod
     66    def _find_profile(cls, project_root: Path) -> t.Optional[Path]:
     67        # TODO Check environment variable
     68        path = Path(project_root, cls.PROFILE_FILE)
     69        if path.exists():
     70            return path
     71
     72        path = Path(Path.home(), ".dbt", cls.PROFILE_FILE)
     73        if path.exists():
     74            return path
     75
     76        return None
     77
     78    @classmethod
     79    def _read_profile(
     80        cls, path: Path, context: DbtContext
     81    ) -> t.Tuple[t.Dict[str, TargetConfig], str]:
     82        with open(path, "r", encoding="utf-8") as file:
     83            source = file.read()
     84        contents = load_yaml(context.render(source))
     85
     86        project_data = contents.get(context.profile_name)
     87        if not project_data:
     88            raise ConfigError(f"Profile '{context.profile_name}' not found in profiles.")
     89
     90        outputs = project_data.get("outputs")
     91        if not outputs:
     92            raise ConfigError(f"No outputs exist in profiles for '{context.profile_name}'.")
     93
     94        targets = {name: TargetConfig.load(name, output) for name, output in outputs.items()}
     95        default_target = context.render(project_data.get("target"))
     96        if default_target not in targets:
     97            raise ConfigError(
     98                f"Default target '{default_target}' not specified in profiles for '{context.profile_name}'."
     99            )
    100
    101        return (targets, default_target)
    102
    103    def to_sqlmesh(self) -> t.Dict[str, ConnectionConfig]:
    104        return {name: target.to_sqlmesh() for name, target in self.targets.items()}

class Profile: View Source

     13class Profile:
     14    """
     15    A class to read DBT profiles and obtain the project's target data warehouse configuration
     16    """
     17
     18    PROFILE_FILE = "profiles.yml"
     19
     20    def __init__(
     21        self,
     22        path: Path,
     23        targets: t.Dict[str, TargetConfig],
     24        default_target: str,
     25    ):
     26        """
     27        Args:
     28            path: Path to the profile file
     29            targets: Dict of targets defined for the project
     30            default_target: Name of the default target for the proejct
     31        """
     32        self.path = path
     33        self.targets = targets
     34        self.default_target = default_target
     35
     36    @classmethod
     37    def load(cls, context: DbtContext) -> Profile:
     38        """
     39        Loads the profile for the specified project
     40
     41        Args:
     42            context: DBT context for this profile
     43
     44        Returns:
     45            The Profile for the specified project
     46        """
     47        if not context.profile_name:
     48            project_file = Path(context.project_root, PROJECT_FILENAME)
     49            if not project_file.exists():
     50                raise ConfigError(f"Could not find {PROJECT_FILENAME} in {context.project_root}")
     51
     52            project_yaml = load_yaml(project_file)
     53            context.profile_name = context.render(
     54                project_yaml.get("profile", "")
     55            ) or context.render(project_yaml.get("name", ""))
     56            if not context.profile_name:
     57                raise ConfigError(f"{project_file.stem} must include project name.")
     58
     59        profile_filepath = cls._find_profile(context.project_root)
     60        if not profile_filepath:
     61            raise ConfigError(f"{cls.PROFILE_FILE} not found.")
     62
     63        targets, default_target = cls._read_profile(profile_filepath, context)
     64        return Profile(profile_filepath, targets, default_target)
     65
     66    @classmethod
     67    def _find_profile(cls, project_root: Path) -> t.Optional[Path]:
     68        # TODO Check environment variable
     69        path = Path(project_root, cls.PROFILE_FILE)
     70        if path.exists():
     71            return path
     72
     73        path = Path(Path.home(), ".dbt", cls.PROFILE_FILE)
     74        if path.exists():
     75            return path
     76
     77        return None
     78
     79    @classmethod
     80    def _read_profile(
     81        cls, path: Path, context: DbtContext
     82    ) -> t.Tuple[t.Dict[str, TargetConfig], str]:
     83        with open(path, "r", encoding="utf-8") as file:
     84            source = file.read()
     85        contents = load_yaml(context.render(source))
     86
     87        project_data = contents.get(context.profile_name)
     88        if not project_data:
     89            raise ConfigError(f"Profile '{context.profile_name}' not found in profiles.")
     90
     91        outputs = project_data.get("outputs")
     92        if not outputs:
     93            raise ConfigError(f"No outputs exist in profiles for '{context.profile_name}'.")
     94
     95        targets = {name: TargetConfig.load(name, output) for name, output in outputs.items()}
     96        default_target = context.render(project_data.get("target"))
     97        if default_target not in targets:
     98            raise ConfigError(
     99                f"Default target '{default_target}' not specified in profiles for '{context.profile_name}'."
    100            )
    101
    102        return (targets, default_target)
    103
    104    def to_sqlmesh(self) -> t.Dict[str, ConnectionConfig]:
    105        return {name: target.to_sqlmesh() for name, target in self.targets.items()}

A class to read DBT profiles and obtain the project's target data warehouse configuration

Profile( path: pathlib.Path, targets: Dict[str, sqlmesh.dbt.target.TargetConfig], default_target: str) View Source

    20    def __init__(
    21        self,
    22        path: Path,
    23        targets: t.Dict[str, TargetConfig],
    24        default_target: str,
    25    ):
    26        """
    27        Args:
    28            path: Path to the profile file
    29            targets: Dict of targets defined for the project
    30            default_target: Name of the default target for the proejct
    31        """
    32        self.path = path
    33        self.targets = targets
    34        self.default_target = default_target

###### Arguments:

- path: Path to the profile file
- targets: Dict of targets defined for the project
- default_target: Name of the default target for the proejct

@classmethod
def load( cls, context: sqlmesh.dbt.common.DbtContext) -> sqlmesh.dbt.profile.Profile: View Source

    36    @classmethod
    37    def load(cls, context: DbtContext) -> Profile:
    38        """
    39        Loads the profile for the specified project
    40
    41        Args:
    42            context: DBT context for this profile
    43
    44        Returns:
    45            The Profile for the specified project
    46        """
    47        if not context.profile_name:
    48            project_file = Path(context.project_root, PROJECT_FILENAME)
    49            if not project_file.exists():
    50                raise ConfigError(f"Could not find {PROJECT_FILENAME} in {context.project_root}")
    51
    52            project_yaml = load_yaml(project_file)
    53            context.profile_name = context.render(
    54                project_yaml.get("profile", "")
    55            ) or context.render(project_yaml.get("name", ""))
    56            if not context.profile_name:
    57                raise ConfigError(f"{project_file.stem} must include project name.")
    58
    59        profile_filepath = cls._find_profile(context.project_root)
    60        if not profile_filepath:
    61            raise ConfigError(f"{cls.PROFILE_FILE} not found.")
    62
    63        targets, default_target = cls._read_profile(profile_filepath, context)
    64        return Profile(profile_filepath, targets, default_target)

Loads the profile for the specified project

###### Arguments:

- context: DBT context for this profile

###### Returns:

> The Profile for the specified project

def to_sqlmesh( self) -> Dict[str, Annotated[Union[sqlmesh.core.config.connection.DuckDBConnectionConfig, sqlmesh.core.config.connection.SnowflakeConnectionConfig, sqlmesh.core.config.connection.DatabricksSQLConnectionConfig, sqlmesh.core.config.connection.DatabricksSparkSessionConnectionConfig, sqlmesh.core.config.connection.DatabricksConnectionConfig, sqlmesh.core.config.connection.BigQueryConnectionConfig, sqlmesh.core.config.connection.RedshiftConnectionConfig], FieldInfo(default=PydanticUndefined, discriminator='type_', extra={})]]: View Source

    104    def to_sqlmesh(self) -> t.Dict[str, ConnectionConfig]:
    105        return {name: target.to_sqlmesh() for name, target in self.targets.items()}