sqlmesh API documentation

Edit on GitHub

SQLMesh is a DataOps framework that brings the best practices of DevOps to data teams. It enables data scientists, analysts, and engineers to efficiently run and deploy data transformations written in SQL or Python.

For more infromation, check out the website and documentation.

## Geting Started

Install SQLMesh through pypi by running:

`pip install sqlmesh`

Follow the tutorial to learn how to use SQLMesh.

## Join our community

We'd love to join you on your data journey. Connect with us in the following ways:

- Join the Tobiko Slack community to ask questions, or just to say hi!
- File an issue on our GitHub.
- Send us an email at hello@tobikodata.com with your questions or feedback.

## Contribution

Contributions in the form of issues or pull requests are greatly appreciated. Read more about how to develop for SQLMesh.

View Source

      1"""
      2.. include:: ../README.md
      3"""
      4from __future__ import annotations
      5
      6import logging
      7import os
      8import sys
      9from enum import Enum
     10
     11from sqlmesh.core.dialect import extend_sqlglot
     12
     13extend_sqlglot()
     14
     15from sqlmesh.core.config import Config
     16from sqlmesh.core.context import Context, ExecutionContext
     17from sqlmesh.core.engine_adapter import EngineAdapter
     18from sqlmesh.core.hooks import hook
     19from sqlmesh.core.macros import macro
     20from sqlmesh.core.model import Model, model
     21from sqlmesh.core.snapshot import Snapshot
     22
     23try:
     24    from sqlmesh._version import __version__, __version_tuple__  # type: ignore
     25except ImportError:
     26    pass
     27
     28
     29class RuntimeEnv(str, Enum):
     30    """Enum defining what environment SQLMesh is running in."""
     31
     32    TERMINAL = "terminal"
     33    DATABRICKS = "databricks"
     34    GOOGLE_COLAB = "google_colab"  # Not currently officially supported
     35    JUPYTER = "jupyter"
     36
     37    @classmethod
     38    def get(cls) -> RuntimeEnv:
     39        """Get the console class to use based on the environment that the code is running in
     40        Reference implementation: https://github.com/noklam/rich/blob/d3a1ae61a77d934844563514370084971bc3e143/rich/console.py#L511-L528
     41
     42        Unlike the rich implementation we try to split out by notebook type instead of treating it all as Jupyter.
     43        """
     44        try:
     45            shell = get_ipython()  # type: ignore
     46            if os.getenv("DATABRICKS_RUNTIME_VERSION"):
     47                return RuntimeEnv.DATABRICKS
     48            if "google.colab" in str(shell.__class__):  # type: ignore
     49                return RuntimeEnv.GOOGLE_COLAB
     50            if shell.__class__.__name__ == "ZMQInteractiveShell":  # type: ignore
     51                return RuntimeEnv.JUPYTER
     52        except NameError:
     53            pass
     54
     55        return RuntimeEnv.TERMINAL
     56
     57    @property
     58    def is_terminal(self) -> bool:
     59        return self == RuntimeEnv.TERMINAL
     60
     61    @property
     62    def is_databricks(self) -> bool:
     63        return self == RuntimeEnv.DATABRICKS
     64
     65    @property
     66    def is_jupyter(self) -> bool:
     67        return self == RuntimeEnv.JUPYTER
     68
     69    @property
     70    def is_google_colab(self) -> bool:
     71        return self == RuntimeEnv.GOOGLE_COLAB
     72
     73    @property
     74    def is_notebook(self) -> bool:
     75        return not self.is_terminal
     76
     77
     78runtime_env = RuntimeEnv.get()
     79
     80
     81if runtime_env.is_notebook:
     82    try:
     83        from sqlmesh.magics import register_magics
     84
     85        register_magics()
     86    except ImportError:
     87        pass
     88
     89
     90# SO: https://stackoverflow.com/questions/384076/how-can-i-color-python-logging-output
     91class CustomFormatter(logging.Formatter):
     92    """Custom logging formatter."""
     93
     94    grey = "\x1b[38;20m"
     95    yellow = "\x1b[33;20m"
     96    red = "\x1b[31;20m"
     97    bold_red = "\x1b[31;1m"
     98    reset = "\x1b[0m"
     99    log_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s (%(filename)s:%(lineno)d)"
    100
    101    FORMATS = {
    102        logging.DEBUG: grey + log_format + reset,
    103        logging.INFO: grey + log_format + reset,
    104        logging.WARNING: yellow + log_format + reset,
    105        logging.ERROR: red + log_format + reset,
    106        logging.CRITICAL: bold_red + log_format + reset,
    107    }
    108
    109    def format(self, record: logging.LogRecord) -> str:
    110        log_fmt = self.FORMATS.get(record.levelno)
    111        formatter = logging.Formatter(log_fmt)
    112        return formatter.format(record)
    113
    114
    115def enable_logging(level: int = logging.INFO) -> None:
    116    """Enable logging to send to stdout and color different levels"""
    117    logger = logging.getLogger()
    118    logger.setLevel(level)
    119    if not logger.hasHandlers():
    120        handler = logging.StreamHandler(sys.stdout)
    121        handler.setLevel(level)
    122        handler.setFormatter(CustomFormatter())
    123        logger.addHandler(handler)

class RuntimeEnv(builtins.str, enum.Enum): View Source

    30class RuntimeEnv(str, Enum):
    31    """Enum defining what environment SQLMesh is running in."""
    32
    33    TERMINAL = "terminal"
    34    DATABRICKS = "databricks"
    35    GOOGLE_COLAB = "google_colab"  # Not currently officially supported
    36    JUPYTER = "jupyter"
    37
    38    @classmethod
    39    def get(cls) -> RuntimeEnv:
    40        """Get the console class to use based on the environment that the code is running in
    41        Reference implementation: https://github.com/noklam/rich/blob/d3a1ae61a77d934844563514370084971bc3e143/rich/console.py#L511-L528
    42
    43        Unlike the rich implementation we try to split out by notebook type instead of treating it all as Jupyter.
    44        """
    45        try:
    46            shell = get_ipython()  # type: ignore
    47            if os.getenv("DATABRICKS_RUNTIME_VERSION"):
    48                return RuntimeEnv.DATABRICKS
    49            if "google.colab" in str(shell.__class__):  # type: ignore
    50                return RuntimeEnv.GOOGLE_COLAB
    51            if shell.__class__.__name__ == "ZMQInteractiveShell":  # type: ignore
    52                return RuntimeEnv.JUPYTER
    53        except NameError:
    54            pass
    55
    56        return RuntimeEnv.TERMINAL
    57
    58    @property
    59    def is_terminal(self) -> bool:
    60        return self == RuntimeEnv.TERMINAL
    61
    62    @property
    63    def is_databricks(self) -> bool:
    64        return self == RuntimeEnv.DATABRICKS
    65
    66    @property
    67    def is_jupyter(self) -> bool:
    68        return self == RuntimeEnv.JUPYTER
    69
    70    @property
    71    def is_google_colab(self) -> bool:
    72        return self == RuntimeEnv.GOOGLE_COLAB
    73
    74    @property
    75    def is_notebook(self) -> bool:
    76        return not self.is_terminal

Enum defining what environment SQLMesh is running in.

TERMINAL = <RuntimeEnv.TERMINAL: 'terminal'>
DATABRICKS = <RuntimeEnv.DATABRICKS: 'databricks'>
GOOGLE_COLAB = <RuntimeEnv.GOOGLE_COLAB: 'google_colab'>
JUPYTER = <RuntimeEnv.JUPYTER: 'jupyter'>
@classmethod
def get(cls) -> sqlmesh.RuntimeEnv: View Source

    38    @classmethod
    39    def get(cls) -> RuntimeEnv:
    40        """Get the console class to use based on the environment that the code is running in
    41        Reference implementation: https://github.com/noklam/rich/blob/d3a1ae61a77d934844563514370084971bc3e143/rich/console.py#L511-L528
    42
    43        Unlike the rich implementation we try to split out by notebook type instead of treating it all as Jupyter.
    44        """
    45        try:
    46            shell = get_ipython()  # type: ignore
    47            if os.getenv("DATABRICKS_RUNTIME_VERSION"):
    48                return RuntimeEnv.DATABRICKS
    49            if "google.colab" in str(shell.__class__):  # type: ignore
    50                return RuntimeEnv.GOOGLE_COLAB
    51            if shell.__class__.__name__ == "ZMQInteractiveShell":  # type: ignore
    52                return RuntimeEnv.JUPYTER
    53        except NameError:
    54            pass
    55
    56        return RuntimeEnv.TERMINAL

Get the console class to use based on the environment that the code is running in Reference implementation: https://github.com/noklam/rich/blob/d3a1ae61a77d934844563514370084971bc3e143/rich/console.py#L511-L528

Unlike the rich implementation we try to split out by notebook type instead of treating it all as Jupyter.

##### Inherited Members

enum.Enum
  name
  value
builtins.str
  encode
  replace
  split
  rsplit
  join
  capitalize
  casefold
  title
  center
  count
  expandtabs
  find
  partition
  index
  ljust
  lower
  lstrip
  rfind
  rindex
  rjust
  rstrip
  rpartition
  splitlines
  strip
  swapcase
  translate
  upper
  startswith
  endswith
  removeprefix
  removesuffix
  isascii
  islower
  isupper
  istitle
  isspace
  isdecimal
  isdigit
  isnumeric
  isalpha
  isalnum
  isidentifier
  isprintable
  zfill
  format
  format_map
  maketrans

runtime_env = <RuntimeEnv.TERMINAL: 'terminal'>
class CustomFormatter(logging.Formatter): View Source

     92class CustomFormatter(logging.Formatter):
     93    """Custom logging formatter."""
     94
     95    grey = "\x1b[38;20m"
     96    yellow = "\x1b[33;20m"
     97    red = "\x1b[31;20m"
     98    bold_red = "\x1b[31;1m"
     99    reset = "\x1b[0m"
    100    log_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s (%(filename)s:%(lineno)d)"
    101
    102    FORMATS = {
    103        logging.DEBUG: grey + log_format + reset,
    104        logging.INFO: grey + log_format + reset,
    105        logging.WARNING: yellow + log_format + reset,
    106        logging.ERROR: red + log_format + reset,
    107        logging.CRITICAL: bold_red + log_format + reset,
    108    }
    109
    110    def format(self, record: logging.LogRecord) -> str:
    111        log_fmt = self.FORMATS.get(record.levelno)
    112        formatter = logging.Formatter(log_fmt)
    113        return formatter.format(record)

Custom logging formatter.

def format(self, record: logging.LogRecord) -> str: View Source

    110    def format(self, record: logging.LogRecord) -> str:
    111        log_fmt = self.FORMATS.get(record.levelno)
    112        formatter = logging.Formatter(log_fmt)
    113        return formatter.format(record)

Format the specified record as text.

The record's attribute dictionary is used as the operand to a string formatting operation which yields the returned string. Before formatting the dictionary, a couple of preparatory steps are carried out. The message attribute of the record is computed using LogRecord.getMessage(). If the formatting string uses the time (as determined by a call to usesTime(), formatTime() is called to format the event time. If there is exception information, it is formatted using formatException() and appended to the message.

##### Inherited Members

logging.Formatter
  Formatter
  converter
  formatTime
  formatException
  usesTime
  formatMessage
  formatStack

def enable_logging(level: int = 20) -> None: View Source

    116def enable_logging(level: int = logging.INFO) -> None:
    117    """Enable logging to send to stdout and color different levels"""
    118    logger = logging.getLogger()
    119    logger.setLevel(level)
    120    if not logger.hasHandlers():
    121        handler = logging.StreamHandler(sys.stdout)
    122        handler.setLevel(level)
    123        handler.setFormatter(CustomFormatter())
    124        logger.addHandler(handler)

Enable logging to send to stdout and color different levels