sqlmesh.core.config.root API documentation

Edit on GitHub

# sqlmesh.core.config.root

View Source

      1from __future__ import annotations
      2
      3import typing as t
      4
      5from pydantic import Field, root_validator, validator
      6
      7from sqlmesh.core import constants as c
      8from sqlmesh.core._typing import NotificationTarget
      9from sqlmesh.core.config.base import BaseConfig, UpdateStrategy
     10from sqlmesh.core.config.categorizer import CategorizerConfig
     11from sqlmesh.core.config.connection import ConnectionConfig, DuckDBConnectionConfig
     12from sqlmesh.core.config.model import ModelDefaultsConfig
     13from sqlmesh.core.config.scheduler import BuiltInSchedulerConfig, SchedulerConfig
     14from sqlmesh.core.loader import Loader, SqlMeshLoader
     15from sqlmesh.core.user import User
     16from sqlmesh.utils.errors import ConfigError
     17
     18
     19class Config(BaseConfig):
     20    """An object used by a Context to configure your SQLMesh project.
     21
     22    Args:
     23        connections: Supported connections and their configurations. Key represents a unique name of a connection.
     24        default_connection: The name of a connection to use by default.
     25        test_connection: The connection settings for tests. Can be a name which refers to an existing configuration
     26            in `connections`.
     27        scheduler: The scheduler configuration.
     28        notification_targets: The notification targets to use.
     29        dialect: The default sql dialect of model queries. Default: same as engine dialect.
     30        physical_schema: The default schema used to store materialized tables.
     31        snapshot_ttl: The period of time that a model snapshot that is not a part of any environment should exist before being deleted.
     32        environment_ttl: The period of time that a development environment should exist before being deleted.
     33        ignore_patterns: Files that match glob patterns specified in this list are ignored when scanning the project folder.
     34        time_column_format: The default format to use for all model time columns. Defaults to %Y-%m-%d.
     35            This time format uses python format codes. https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes.
     36        auto_categorize_changes: Indicates whether SQLMesh should attempt to automatically categorize model changes (breaking / non-breaking)
     37            during plan creation.
     38        users: A list of users that can be used for approvals/notifications.
     39        model_defaults: Default values for model definitions.
     40    """
     41
     42    connections: t.Union[t.Dict[str, ConnectionConfig], ConnectionConfig] = DuckDBConnectionConfig()
     43    default_connection: str = ""
     44    test_connection_: t.Union[ConnectionConfig, str] = Field(
     45        alias="test_connection", default=DuckDBConnectionConfig()
     46    )
     47    scheduler: SchedulerConfig = BuiltInSchedulerConfig()
     48    notification_targets: t.List[NotificationTarget] = []
     49    physical_schema: str = ""
     50    snapshot_ttl: str = c.DEFAULT_SNAPSHOT_TTL
     51    environment_ttl: t.Optional[str] = c.DEFAULT_ENVIRONMENT_TTL
     52    ignore_patterns: t.List[str] = []
     53    time_column_format: str = c.DEFAULT_TIME_COLUMN_FORMAT
     54    auto_categorize_changes: CategorizerConfig = CategorizerConfig()
     55    users: t.List[User] = []
     56    model_defaults: ModelDefaultsConfig = ModelDefaultsConfig()
     57    loader: t.Type[Loader] = SqlMeshLoader
     58
     59    _FIELD_UPDATE_STRATEGY: t.ClassVar[t.Dict[str, UpdateStrategy]] = {
     60        "connections": UpdateStrategy.KEY_UPDATE,
     61        "notification_targets": UpdateStrategy.EXTEND,
     62        "ignore_patterns": UpdateStrategy.EXTEND,
     63        "users": UpdateStrategy.EXTEND,
     64        "model_defaults": UpdateStrategy.NESTED_UPDATE,
     65        "auto_categorize_changes": UpdateStrategy.NESTED_UPDATE,
     66    }
     67
     68    @validator("connections", always=True)
     69    def _connections_ensure_dict(
     70        cls, value: t.Union[t.Dict[str, ConnectionConfig], ConnectionConfig]
     71    ) -> t.Dict[str, ConnectionConfig]:
     72        if not isinstance(value, dict):
     73            return {"": value}
     74        return value
     75
     76    @root_validator(pre=True)
     77    def _normalize_fields(cls, values: t.Dict[str, t.Any]) -> t.Dict[str, t.Any]:
     78        if "connections" not in values and "connection" in values:
     79            values["connections"] = values.pop("connection")
     80
     81        return values
     82
     83    def get_connection(self, name: t.Optional[str] = None) -> ConnectionConfig:
     84        if isinstance(self.connections, dict):
     85            if name is None:
     86                if self.default_connection:
     87                    if self.default_connection not in self.connections:
     88                        raise ConfigError(
     89                            f"Missing connection with name '{self.default_connection}'"
     90                        )
     91                    return self.connections[self.default_connection]
     92
     93                if "" in self.connections:
     94                    return self.connections[""]
     95
     96                return next(iter(self.connections.values()))
     97
     98            if name not in self.connections:
     99                raise ConfigError(f"Missing connection with name '{name}'.")
    100
    101            return self.connections[name]
    102        else:
    103            if name is not None:
    104                raise ConfigError(
    105                    "Connection name is not supported when only one connection is configured."
    106                )
    107            return self.connections
    108
    109    @property
    110    def test_connection(self) -> ConnectionConfig:
    111        if isinstance(self.test_connection_, str):
    112            return self.get_connection(self.test_connection_)
    113        return self.test_connection_

class Config(sqlmesh.core.config.base.BaseConfig): View Source

     20class Config(BaseConfig):
     21    """An object used by a Context to configure your SQLMesh project.
     22
     23    Args:
     24        connections: Supported connections and their configurations. Key represents a unique name of a connection.
     25        default_connection: The name of a connection to use by default.
     26        test_connection: The connection settings for tests. Can be a name which refers to an existing configuration
     27            in `connections`.
     28        scheduler: The scheduler configuration.
     29        notification_targets: The notification targets to use.
     30        dialect: The default sql dialect of model queries. Default: same as engine dialect.
     31        physical_schema: The default schema used to store materialized tables.
     32        snapshot_ttl: The period of time that a model snapshot that is not a part of any environment should exist before being deleted.
     33        environment_ttl: The period of time that a development environment should exist before being deleted.
     34        ignore_patterns: Files that match glob patterns specified in this list are ignored when scanning the project folder.
     35        time_column_format: The default format to use for all model time columns. Defaults to %Y-%m-%d.
     36            This time format uses python format codes. https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes.
     37        auto_categorize_changes: Indicates whether SQLMesh should attempt to automatically categorize model changes (breaking / non-breaking)
     38            during plan creation.
     39        users: A list of users that can be used for approvals/notifications.
     40        model_defaults: Default values for model definitions.
     41    """
     42
     43    connections: t.Union[t.Dict[str, ConnectionConfig], ConnectionConfig] = DuckDBConnectionConfig()
     44    default_connection: str = ""
     45    test_connection_: t.Union[ConnectionConfig, str] = Field(
     46        alias="test_connection", default=DuckDBConnectionConfig()
     47    )
     48    scheduler: SchedulerConfig = BuiltInSchedulerConfig()
     49    notification_targets: t.List[NotificationTarget] = []
     50    physical_schema: str = ""
     51    snapshot_ttl: str = c.DEFAULT_SNAPSHOT_TTL
     52    environment_ttl: t.Optional[str] = c.DEFAULT_ENVIRONMENT_TTL
     53    ignore_patterns: t.List[str] = []
     54    time_column_format: str = c.DEFAULT_TIME_COLUMN_FORMAT
     55    auto_categorize_changes: CategorizerConfig = CategorizerConfig()
     56    users: t.List[User] = []
     57    model_defaults: ModelDefaultsConfig = ModelDefaultsConfig()
     58    loader: t.Type[Loader] = SqlMeshLoader
     59
     60    _FIELD_UPDATE_STRATEGY: t.ClassVar[t.Dict[str, UpdateStrategy]] = {
     61        "connections": UpdateStrategy.KEY_UPDATE,
     62        "notification_targets": UpdateStrategy.EXTEND,
     63        "ignore_patterns": UpdateStrategy.EXTEND,
     64        "users": UpdateStrategy.EXTEND,
     65        "model_defaults": UpdateStrategy.NESTED_UPDATE,
     66        "auto_categorize_changes": UpdateStrategy.NESTED_UPDATE,
     67    }
     68
     69    @validator("connections", always=True)
     70    def _connections_ensure_dict(
     71        cls, value: t.Union[t.Dict[str, ConnectionConfig], ConnectionConfig]
     72    ) -> t.Dict[str, ConnectionConfig]:
     73        if not isinstance(value, dict):
     74            return {"": value}
     75        return value
     76
     77    @root_validator(pre=True)
     78    def _normalize_fields(cls, values: t.Dict[str, t.Any]) -> t.Dict[str, t.Any]:
     79        if "connections" not in values and "connection" in values:
     80            values["connections"] = values.pop("connection")
     81
     82        return values
     83
     84    def get_connection(self, name: t.Optional[str] = None) -> ConnectionConfig:
     85        if isinstance(self.connections, dict):
     86            if name is None:
     87                if self.default_connection:
     88                    if self.default_connection not in self.connections:
     89                        raise ConfigError(
     90                            f"Missing connection with name '{self.default_connection}'"
     91                        )
     92                    return self.connections[self.default_connection]
     93
     94                if "" in self.connections:
     95                    return self.connections[""]
     96
     97                return next(iter(self.connections.values()))
     98
     99            if name not in self.connections:
    100                raise ConfigError(f"Missing connection with name '{name}'.")
    101
    102            return self.connections[name]
    103        else:
    104            if name is not None:
    105                raise ConfigError(
    106                    "Connection name is not supported when only one connection is configured."
    107                )
    108            return self.connections
    109
    110    @property
    111    def test_connection(self) -> ConnectionConfig:
    112        if isinstance(self.test_connection_, str):
    113            return self.get_connection(self.test_connection_)
    114        return self.test_connection_

An object used by a Context to configure your SQLMesh project.

###### Arguments:

- connections: Supported connections and their configurations. Key represents a unique name of a connection.
- default_connection: The name of a connection to use by default.
- test_connection: The connection settings for tests. Can be a name which refers to an existing configuration in `connections`.
- scheduler: The scheduler configuration.
- notification_targets: The notification targets to use.
- dialect: The default sql dialect of model queries. Default: same as engine dialect.
- physical_schema: The default schema used to store materialized tables.
- snapshot_ttl: The period of time that a model snapshot that is not a part of any environment should exist before being deleted.
- environment_ttl: The period of time that a development environment should exist before being deleted.
- ignore_patterns: Files that match glob patterns specified in this list are ignored when scanning the project folder.
- time_column_format: The default format to use for all model time columns. Defaults to %Y-%m-%d. This time format uses python format codes. https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes.
- auto_categorize_changes: Indicates whether SQLMesh should attempt to automatically categorize model changes (breaking / non-breaking) during plan creation.
- users: A list of users that can be used for approvals/notifications.
- model_defaults: Default values for model definitions.

def get_connection( self, name: Optional[str] = None) -> Annotated[Union[sqlmesh.core.config.connection.DuckDBConnectionConfig, sqlmesh.core.config.connection.SnowflakeConnectionConfig, sqlmesh.core.config.connection.DatabricksSQLConnectionConfig, sqlmesh.core.config.connection.DatabricksSparkSessionConnectionConfig, sqlmesh.core.config.connection.DatabricksConnectionConfig, sqlmesh.core.config.connection.BigQueryConnectionConfig, sqlmesh.core.config.connection.RedshiftConnectionConfig], FieldInfo(default=PydanticUndefined, discriminator='type_', extra={})]: View Source

     84    def get_connection(self, name: t.Optional[str] = None) -> ConnectionConfig:
     85        if isinstance(self.connections, dict):
     86            if name is None:
     87                if self.default_connection:
     88                    if self.default_connection not in self.connections:
     89                        raise ConfigError(
     90                            f"Missing connection with name '{self.default_connection}'"
     91                        )
     92                    return self.connections[self.default_connection]
     93
     94                if "" in self.connections:
     95                    return self.connections[""]
     96
     97                return next(iter(self.connections.values()))
     98
     99            if name not in self.connections:
    100                raise ConfigError(f"Missing connection with name '{name}'.")
    101
    102            return self.connections[name]
    103        else:
    104            if name is not None:
    105                raise ConfigError(
    106                    "Connection name is not supported when only one connection is configured."
    107                )
    108            return self.connections

##### Inherited Members

pydantic.main.BaseModel
  BaseModel
  parse_obj
  parse_raw
  parse_file
  from_orm
  construct
  copy
  schema
  schema_json
  validate
  update_forward_refs
sqlmesh.core.config.base.BaseConfig
  update_with
sqlmesh.utils.pydantic.PydanticModel
  Config
  dict
  json
  missing_required_fields
  extra_fields
  all_fields
  required_fields