sqlmesh.core.config.loader API documentation

Edit on GitHub

# sqlmesh.core.config.loader

View Source

     1from __future__ import annotations
     2
     3import os
     4import typing as t
     5from pathlib import Path
     6
     7from sqlmesh.core import constants as c
     8from sqlmesh.core.config.root import Config
     9from sqlmesh.utils import sys_path
    10from sqlmesh.utils.errors import ConfigError
    11from sqlmesh.utils.metaprogramming import import_python_file
    12from sqlmesh.utils.yaml import load as yaml_load
    13
    14
    15def load_config_from_paths(
    16    *paths: Path, config_name: str = "config", load_from_env: bool = True
    17) -> Config:
    18    config: t.Optional[Config] = None
    19
    20    visited_folders: t.Set[Path] = set()
    21    for path in paths:
    22        if not path.exists():
    23            continue
    24
    25        if not path.is_file():
    26            raise ConfigError(f"Path '{path}' must be a file.")
    27
    28        parent_path = path.parent
    29        if parent_path in visited_folders:
    30            raise ConfigError(f"Multiple configuration files found in folder '{parent_path}'.")
    31        visited_folders.add(parent_path)
    32
    33        extension = path.name.split(".")[-1].lower()
    34        if extension in ("yml", "yaml"):
    35            next_config = load_config_from_yaml(path)
    36        elif extension == "py":
    37            next_config = load_config_from_python_module(path, config_name=config_name)
    38        else:
    39            raise ConfigError(
    40                f"Unsupported config file extension '{extension}' in config file '{path}'."
    41            )
    42
    43        config = config.update_with(next_config) if config is not None else next_config
    44
    45    if config is None:
    46        raise ConfigError(
    47            "SQLMesh config could not be found. Point the cli to the right path with `sqlmesh --path`. If you haven't set up SQLMesh, run `sqlmesh init`."
    48        )
    49
    50    if load_from_env:
    51        config = config.update_with(load_config_from_env())
    52
    53    return config
    54
    55
    56def load_config_from_yaml(path: Path) -> Config:
    57    config_dict = yaml_load(path)
    58    return Config.parse_obj(config_dict)
    59
    60
    61def load_config_from_python_module(module_path: Path, config_name: str = "config") -> Config:
    62    with sys_path(module_path.parent):
    63        try:
    64            config_module = import_python_file(module_path, module_path.parent)
    65        except ImportError:
    66            raise ConfigError(f"Config module '{module_path}' was not found.")
    67
    68    try:
    69        config_obj = getattr(config_module, config_name)
    70    except AttributeError:
    71        raise ConfigError(f"Config '{config_name}' was not found.")
    72
    73    if config_obj is None or not isinstance(config_obj, Config):
    74        raise ConfigError(
    75            f"Config needs to be a valid object of type sqlmesh.core.config.Config. Found `{config_obj}` instead at '{module_path}'."
    76        )
    77
    78    return config_obj
    79
    80
    81def load_config_from_env() -> Config:
    82    config_dict: t.Dict[str, t.Any] = {}
    83
    84    for key, value in os.environ.items():
    85        key = key.lower()
    86        if key.startswith(f"{c.SQLMESH}__"):
    87            segments = key.split("__")[1:]
    88            if not segments or not segments[-1]:
    89                raise ConfigError(f"Invalid SQLMesh configuration variable '{key}'.")
    90
    91            target_dict = config_dict
    92            for config_key in segments[:-1]:
    93                if config_key not in target_dict:
    94                    target_dict[config_key] = {}
    95                target_dict = target_dict[config_key]
    96            target_dict[segments[-1]] = value
    97
    98    return Config.parse_obj(config_dict)

def load_config_from_paths( *paths: pathlib.Path, config_name: str = 'config', load_from_env: bool = True) -> sqlmesh.core.config.root.Config: View Source

    16def load_config_from_paths(
    17    *paths: Path, config_name: str = "config", load_from_env: bool = True
    18) -> Config:
    19    config: t.Optional[Config] = None
    20
    21    visited_folders: t.Set[Path] = set()
    22    for path in paths:
    23        if not path.exists():
    24            continue
    25
    26        if not path.is_file():
    27            raise ConfigError(f"Path '{path}' must be a file.")
    28
    29        parent_path = path.parent
    30        if parent_path in visited_folders:
    31            raise ConfigError(f"Multiple configuration files found in folder '{parent_path}'.")
    32        visited_folders.add(parent_path)
    33
    34        extension = path.name.split(".")[-1].lower()
    35        if extension in ("yml", "yaml"):
    36            next_config = load_config_from_yaml(path)
    37        elif extension == "py":
    38            next_config = load_config_from_python_module(path, config_name=config_name)
    39        else:
    40            raise ConfigError(
    41                f"Unsupported config file extension '{extension}' in config file '{path}'."
    42            )
    43
    44        config = config.update_with(next_config) if config is not None else next_config
    45
    46    if config is None:
    47        raise ConfigError(
    48            "SQLMesh config could not be found. Point the cli to the right path with `sqlmesh --path`. If you haven't set up SQLMesh, run `sqlmesh init`."
    49        )
    50
    51    if load_from_env:
    52        config = config.update_with(load_config_from_env())
    53
    54    return config

def load_config_from_yaml(path: pathlib.Path) -> sqlmesh.core.config.root.Config: View Source

    57def load_config_from_yaml(path: Path) -> Config:
    58    config_dict = yaml_load(path)
    59    return Config.parse_obj(config_dict)

def load_config_from_python_module( module_path: pathlib.Path, config_name: str = 'config') -> sqlmesh.core.config.root.Config: View Source

    62def load_config_from_python_module(module_path: Path, config_name: str = "config") -> Config:
    63    with sys_path(module_path.parent):
    64        try:
    65            config_module = import_python_file(module_path, module_path.parent)
    66        except ImportError:
    67            raise ConfigError(f"Config module '{module_path}' was not found.")
    68
    69    try:
    70        config_obj = getattr(config_module, config_name)
    71    except AttributeError:
    72        raise ConfigError(f"Config '{config_name}' was not found.")
    73
    74    if config_obj is None or not isinstance(config_obj, Config):
    75        raise ConfigError(
    76            f"Config needs to be a valid object of type sqlmesh.core.config.Config. Found `{config_obj}` instead at '{module_path}'."
    77        )
    78
    79    return config_obj

def load_config_from_env() -> sqlmesh.core.config.root.Config: View Source

    82def load_config_from_env() -> Config:
    83    config_dict: t.Dict[str, t.Any] = {}
    84
    85    for key, value in os.environ.items():
    86        key = key.lower()
    87        if key.startswith(f"{c.SQLMESH}__"):
    88            segments = key.split("__")[1:]
    89            if not segments or not segments[-1]:
    90                raise ConfigError(f"Invalid SQLMesh configuration variable '{key}'.")
    91
    92            target_dict = config_dict
    93            for config_key in segments[:-1]:
    94                if config_key not in target_dict:
    95                    target_dict[config_key] = {}
    96                target_dict = target_dict[config_key]
    97            target_dict[segments[-1]] = value
    98
    99    return Config.parse_obj(config_dict)