sqlmesh.cli.example_project API documentation

Edit on GitHub

# sqlmesh.cli.example_project

View Source

      1import typing as t
      2from enum import Enum
      3from pathlib import Path
      4
      5import click
      6
      7DEFAULT_CONFIG = """connections:
      8    local:
      9        type: duckdb
     10        database: db.duckdb
     11
     12default_connection: local
     13"""
     14
     15
     16DEFAULT_AIRFLOW_CONFIG = """scheduler:
     17    type: airflow
     18    airflow_url: http://localhost:8080/
     19    username: airflow
     20    password: airflow
     21"""
     22
     23DEFAULT_DBT_CONFIG = """from pathlib import Path
     24
     25from sqlmesh.dbt.loader import sqlmesh_config
     26
     27config = sqlmesh_config(Path(__file__).parent)
     28"""
     29
     30EXAMPLE_SCHEMA_NAME = "sqlmesh_example"
     31EXAMPLE_FULL_MODEL_NAME = f"{EXAMPLE_SCHEMA_NAME}.example_full_model"
     32EXAMPLE_INCREMENTAL_MODEL_NAME = f"{EXAMPLE_SCHEMA_NAME}.example_incremental_model"
     33
     34
     35EXAMPLE_FULL_MODEL_DEF = f"""MODEL (
     36  name {EXAMPLE_FULL_MODEL_NAME},
     37  kind FULL,
     38  cron '@daily',
     39  audits [assert_positive_order_ids],
     40);
     41
     42SELECT
     43  item_id,
     44  count(distinct id) AS num_orders,
     45FROM
     46    {EXAMPLE_INCREMENTAL_MODEL_NAME}
     47GROUP BY item_id
     48"""
     49
     50EXAMPLE_INCREMENTAL_MODEL_DEF = f"""MODEL (
     51    name {EXAMPLE_INCREMENTAL_MODEL_NAME},
     52    kind INCREMENTAL_BY_TIME_RANGE (
     53        time_column ds
     54    ),
     55    start '2020-01-01',
     56    cron '@daily',
     57);
     58
     59SELECT
     60    id,
     61    item_id,
     62    ds,
     63FROM
     64    (VALUES
     65        (1, 1, '2020-01-01'),
     66        (1, 2, '2020-01-01'),
     67        (2, 1, '2020-01-01'),
     68        (3, 3, '2020-01-03'),
     69        (4, 1, '2020-01-04'),
     70        (5, 1, '2020-01-05'),
     71        (6, 1, '2020-01-06'),
     72        (7, 1, '2020-01-07')
     73    ) AS t (id, item_id, ds)
     74WHERE
     75    ds between @start_ds and @end_ds
     76"""
     77
     78EXAMPLE_AUDIT = f"""AUDIT (
     79  name assert_positive_order_ids,
     80);
     81
     82SELECT *
     83FROM @this_model
     84WHERE
     85  item_id < 0
     86"""
     87
     88
     89EXAMPLE_TEST = f"""test_example_full_model:
     90  model: {EXAMPLE_FULL_MODEL_NAME}
     91  inputs:
     92    {EXAMPLE_INCREMENTAL_MODEL_NAME}:
     93        rows:
     94        - id: 1
     95          item_id: 1
     96          ds: '2020-01-01'
     97        - id: 2
     98          item_id: 1
     99          ds: '2020-01-02'
    100        - id: 3
    101          item_id: 2
    102          ds: '2020-01-03'
    103  outputs:
    104    query:
    105      rows:
    106      - item_id: 1
    107        num_orders: 2
    108      - item_id: 2
    109        num_orders: 1
    110"""
    111
    112
    113class ProjectTemplate(Enum):
    114    AIRFLOW = "airflow"
    115    DBT = "dbt"
    116    DEFAULT = "default"
    117
    118
    119DEFAULT_CONFIGS = {
    120    ProjectTemplate.AIRFLOW: DEFAULT_AIRFLOW_CONFIG,
    121    ProjectTemplate.DBT: DEFAULT_DBT_CONFIG,
    122    ProjectTemplate.DEFAULT: DEFAULT_CONFIG,
    123}
    124
    125
    126def init_example_project(
    127    path: t.Union[str, Path], template: ProjectTemplate = ProjectTemplate.DEFAULT
    128) -> None:
    129    root_path = Path(path)
    130    config_extension = "py" if template == ProjectTemplate.DBT else "yaml"
    131    config_path = root_path / f"config.{config_extension}"
    132    audits_path = root_path / "audits"
    133    macros_path = root_path / "macros"
    134    models_path = root_path / "models"
    135    tests_path = root_path / "tests"
    136
    137    if config_path.exists():
    138        raise click.ClickException(f"Found an existing config in '{config_path}'")
    139
    140    _create_config(config_path, template)
    141    if template == ProjectTemplate.DBT:
    142        return
    143
    144    _create_folders([audits_path, macros_path, models_path, tests_path])
    145    _create_macros(macros_path)
    146    _create_audits(audits_path)
    147    _create_models(models_path)
    148    _create_tests(tests_path)
    149
    150
    151def _create_folders(target_folders: t.Sequence[Path]) -> None:
    152    for folder_path in target_folders:
    153        folder_path.mkdir(exist_ok=True)
    154        (folder_path / ".gitkeep").touch()
    155
    156
    157def _create_config(config_path: Path, template: ProjectTemplate) -> None:
    158    _write_file(
    159        config_path,
    160        DEFAULT_CONFIGS[template],
    161    )
    162
    163
    164def _create_macros(macros_path: Path) -> None:
    165    (macros_path / "__init__.py").touch()
    166
    167
    168def _create_audits(audits_path: Path) -> None:
    169    _write_file(audits_path / "example_full_model.sql", EXAMPLE_AUDIT)
    170
    171
    172def _create_models(models_path: Path) -> None:
    173    for model_name, model_def in [
    174        (EXAMPLE_FULL_MODEL_NAME, EXAMPLE_FULL_MODEL_DEF),
    175        (EXAMPLE_INCREMENTAL_MODEL_NAME, EXAMPLE_INCREMENTAL_MODEL_DEF),
    176    ]:
    177        _write_file(models_path / f"{model_name.split('.')[-1]}.sql", model_def)
    178
    179
    180def _create_tests(tests_path: Path) -> None:
    181    _write_file(tests_path / "test_example_full_model.yaml", EXAMPLE_TEST)
    182
    183
    184def _write_file(path: Path, payload: str) -> None:
    185    with open(path, "w", encoding="utf-8") as fd:
    186        fd.write(payload)

class ProjectTemplate(enum.Enum): View Source

    114class ProjectTemplate(Enum):
    115    AIRFLOW = "airflow"
    116    DBT = "dbt"
    117    DEFAULT = "default"

An enumeration.

AIRFLOW = <ProjectTemplate.AIRFLOW: 'airflow'>
DBT = <ProjectTemplate.DBT: 'dbt'>
DEFAULT = <ProjectTemplate.DEFAULT: 'default'>

##### Inherited Members

enum.Enum
  name
  value

def init_example_project( path: Union[str, pathlib.Path], template: sqlmesh.cli.example_project.ProjectTemplate = <ProjectTemplate.DEFAULT: 'default'>) -> None: View Source

    127def init_example_project(
    128    path: t.Union[str, Path], template: ProjectTemplate = ProjectTemplate.DEFAULT
    129) -> None:
    130    root_path = Path(path)
    131    config_extension = "py" if template == ProjectTemplate.DBT else "yaml"
    132    config_path = root_path / f"config.{config_extension}"
    133    audits_path = root_path / "audits"
    134    macros_path = root_path / "macros"
    135    models_path = root_path / "models"
    136    tests_path = root_path / "tests"
    137
    138    if config_path.exists():
    139        raise click.ClickException(f"Found an existing config in '{config_path}'")
    140
    141    _create_config(config_path, template)
    142    if template == ProjectTemplate.DBT:
    143        return
    144
    145    _create_folders([audits_path, macros_path, models_path, tests_path])
    146    _create_macros(macros_path)
    147    _create_audits(audits_path)
    148    _create_models(models_path)
    149    _create_tests(tests_path)