sqlmesh.utils.metaprogramming API documentation

Edit on GitHub

# sqlmesh.utils.metaprogramming

View Source

      1from __future__ import annotations
      2
      3import ast
      4import dis
      5import importlib
      6import inspect
      7import linecache
      8import os
      9import re
     10import sys
     11import textwrap
     12import types
     13import typing as t
     14from enum import Enum
     15from pathlib import Path
     16
     17from astor import to_source
     18
     19from sqlmesh.utils import format_exception, unique
     20from sqlmesh.utils.errors import SQLMeshError
     21from sqlmesh.utils.pydantic import PydanticModel
     22
     23IGNORE_DECORATORS = {"hook", "macro", "model"}
     24
     25
     26def _is_relative_to(path: t.Optional[Path | str], other: t.Optional[Path | str]) -> bool:
     27    """path.is_relative_to compatibility, was only supported >= 3.9"""
     28    if path is None or other is None:
     29        return False
     30
     31    if isinstance(path, str):
     32        path = Path(path)
     33    if isinstance(other, str):
     34        other = Path(other)
     35
     36    try:
     37        path.absolute().relative_to(other.absolute())
     38        return True
     39    except ValueError:
     40        return False
     41
     42
     43def _code_globals(code: types.CodeType) -> t.Dict[str, None]:
     44    variables = {
     45        instruction.argval: None
     46        for instruction in dis.get_instructions(code)
     47        if instruction.opname == "LOAD_GLOBAL"
     48    }
     49
     50    for const in code.co_consts:
     51        if isinstance(const, types.CodeType):
     52            variables.update(_code_globals(const))
     53
     54    return variables
     55
     56
     57def func_globals(func: t.Callable) -> t.Dict[str, t.Any]:
     58    """Finds all global references and closures in a function and nested functions.
     59
     60    This function treats closures as global variables, which could cause problems in the future.
     61
     62    Args:
     63        func: The function to introspect
     64
     65    Returns:
     66        A dictionary of all global references.
     67    """
     68    variables = {}
     69
     70    if hasattr(func, "__code__"):
     71        code = func.__code__
     72
     73        for var in list(_code_globals(code)) + decorators(func):
     74            if var in func.__globals__:
     75                ref = func.__globals__[var]
     76                variables[var] = ref
     77
     78        if func.__closure__:
     79            for var, value in zip(code.co_freevars, func.__closure__):
     80                variables[var] = value.cell_contents
     81
     82    return variables
     83
     84
     85class ClassFoundException(Exception):
     86    pass
     87
     88
     89class _ClassFinder(ast.NodeVisitor):
     90    def __init__(self, qualname: str) -> None:
     91        self.stack: t.List[str] = []
     92        self.qualname = qualname
     93
     94    def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
     95        self.stack.append(node.name)
     96        self.stack.append("<locals>")
     97        self.generic_visit(node)
     98        self.stack.pop()
     99        self.stack.pop()
    100
    101    visit_AsyncFunctionDef = visit_FunctionDef  # type: ignore
    102
    103    def visit_ClassDef(self, node: ast.ClassDef) -> None:
    104        self.stack.append(node.name)
    105        if self.qualname == ".".join(self.stack):
    106            # Return the decorator for the class if present
    107            if node.decorator_list:
    108                line_number = node.decorator_list[0].lineno
    109            else:
    110                line_number = node.lineno
    111
    112            # decrement by one since lines starts with indexing by zero
    113            line_number -= 1
    114            raise ClassFoundException(line_number)
    115        self.generic_visit(node)
    116        self.stack.pop()
    117
    118
    119def getsource(obj: t.Any) -> str:
    120    """Get the source of a function or class.
    121
    122    inspect.getsource doesn't find decorators in python < 3.9
    123    https://github.com/python/cpython/commit/696136b993e11b37c4f34d729a0375e5ad544ade
    124    """
    125    path = inspect.getsourcefile(obj)
    126    if path:
    127        module = inspect.getmodule(obj, path)
    128
    129        if module:
    130            lines = linecache.getlines(path, module.__dict__)
    131        else:
    132            lines = linecache.getlines(path)
    133
    134        def join_source(lnum: int) -> str:
    135            return "".join(inspect.getblock(lines[lnum:]))
    136
    137        if inspect.isclass(obj):
    138            qualname = obj.__qualname__
    139            source = "".join(lines)
    140            tree = ast.parse(source)
    141            class_finder = _ClassFinder(qualname)
    142            try:
    143                class_finder.visit(tree)
    144            except ClassFoundException as e:
    145                return join_source(e.args[0])
    146        elif inspect.isfunction(obj):
    147            obj = obj.__code__
    148            if hasattr(obj, "co_firstlineno"):
    149                lnum = obj.co_firstlineno - 1
    150                pat = re.compile(r"^(\s*def\s)|(\s*async\s+def\s)|(.*(?<!\w)lambda(:|\s))|^(\s*@)")
    151                while lnum > 0:
    152                    try:
    153                        line = lines[lnum]
    154                    except IndexError:
    155                        raise OSError("lineno is out of bounds")
    156                    if pat.match(line):
    157                        break
    158                    lnum = lnum - 1
    159                return join_source(lnum)
    160    raise SQLMeshError(f"Cannot find source for {obj}")
    161
    162
    163def parse_source(func: t.Callable) -> ast.Module:
    164    """Parse a function and returns an ast node."""
    165    return ast.parse(textwrap.dedent(getsource(func)))
    166
    167
    168def _decorator_name(decorator: ast.expr) -> str:
    169    if isinstance(decorator, ast.Call):
    170        return decorator.func.id  # type: ignore
    171    if isinstance(decorator, ast.Name):
    172        return decorator.id
    173    return ""
    174
    175
    176def decorators(func: t.Callable) -> t.List[str]:
    177    """Finds a list of all the decorators of a callable."""
    178    root_node = parse_source(func)
    179    decorators = []
    180
    181    for node in ast.walk(root_node):
    182        if isinstance(node, (ast.FunctionDef, ast.ClassDef)):
    183            for decorator in node.decorator_list:
    184                name = _decorator_name(decorator)
    185                if name not in IGNORE_DECORATORS:
    186                    decorators.append(name)
    187    return unique(decorators)
    188
    189
    190def normalize_source(obj: t.Any) -> str:
    191    """Rewrites an object's source with formatting and doc strings removed by using Python ast.
    192
    193    Args:
    194        obj: The object to fetch source from and convert to a string.
    195
    196    Returns:
    197        A string representation of the normalized function.
    198    """
    199    root_node = parse_source(obj)
    200
    201    for node in ast.walk(root_node):
    202        if isinstance(node, (ast.FunctionDef, ast.ClassDef)):
    203            for decorator in node.decorator_list:
    204                if _decorator_name(decorator) in IGNORE_DECORATORS:
    205                    node.decorator_list.remove(decorator)
    206
    207            # remove docstrings
    208            body = node.body
    209            if body and isinstance(body[0], ast.Expr) and isinstance(body[0].value, ast.Str):
    210                node.body = body[1:]
    211
    212            # remove function return type annotation
    213            if isinstance(node, ast.FunctionDef):
    214                node.returns = None
    215        elif isinstance(node, ast.arg):
    216            node.annotation = None
    217
    218    return to_source(root_node).strip()
    219
    220
    221def build_env(
    222    obj: t.Any,
    223    *,
    224    env: t.Dict[str, t.Any],
    225    name: str,
    226    path: Path,
    227) -> None:
    228    """Fills in env dictionary with all globals needed to execute the object.
    229
    230    Recursively traverse classes and functions.
    231
    232    Args:
    233        obj: Any python object.
    234        env: Dictionary to store the env.
    235        name: Name of the object in the env.
    236        path: The module path to serialize. Other modules will not be walked and treated as imports.
    237    """
    238
    239    obj_module = inspect.getmodule(obj)
    240
    241    if obj_module and obj_module.__name__ == "builtins":
    242        return
    243
    244    def walk(obj: t.Any) -> None:
    245        if inspect.isclass(obj):
    246            for decorator in decorators(obj):
    247                if obj_module and decorator in obj_module.__dict__:
    248                    build_env(
    249                        obj_module.__dict__[decorator],
    250                        env=env,
    251                        name=decorator,
    252                        path=path,
    253                    )
    254
    255            for base in obj.__bases__:
    256                build_env(base, env=env, name=base.__qualname__, path=path)
    257
    258            for k, v in obj.__dict__.items():
    259                if k.startswith("__"):
    260                    continue
    261                # traverse methods in a class to find global references
    262                if isinstance(v, (classmethod, staticmethod)):
    263                    v = v.__func__
    264                if callable(v):
    265                    # if the method is a part of the object, walk it
    266                    # else it is a global function and we just store it
    267                    if v.__qualname__.startswith(obj.__qualname__):
    268                        walk(v)
    269                    else:
    270                        build_env(v, env=env, name=v.__name__, path=path)
    271        elif callable(obj):
    272            for k, v in func_globals(obj).items():
    273                build_env(v, env=env, name=k, path=path)
    274
    275    if name not in env:
    276        env[name] = obj
    277        if obj_module and _is_relative_to(obj_module.__file__, path):
    278            walk(obj)
    279    elif env[name] != obj:
    280        raise SQLMeshError(
    281            f"Cannot store {obj} in environment, duplicate definitions found for '{name}'"
    282        )
    283
    284
    285class ExecutableKind(str, Enum):
    286    """The kind of of executable. The order of the members is used when serializing the python model to text."""
    287
    288    IMPORT = "import"
    289    VALUE = "value"
    290    DEFINITION = "definition"
    291    STATEMENT = "statement"
    292
    293    def __lt__(self, other: t.Any) -> bool:
    294        if not isinstance(other, ExecutableKind):
    295            return NotImplemented
    296        values = list(ExecutableKind.__dict__.values())
    297        return values.index(self) < values.index(other)
    298
    299
    300class Executable(PydanticModel):
    301    payload: t.Any
    302    kind: ExecutableKind = ExecutableKind.DEFINITION
    303    name: t.Optional[str] = None
    304    path: t.Optional[str] = None
    305    alias: t.Optional[str] = None
    306
    307    @property
    308    def is_definition(self) -> bool:
    309        return self.kind == ExecutableKind.DEFINITION
    310
    311    @property
    312    def is_import(self) -> bool:
    313        return self.kind == ExecutableKind.IMPORT
    314
    315    @property
    316    def is_statement(self) -> bool:
    317        return self.kind == ExecutableKind.STATEMENT
    318
    319    @property
    320    def is_value(self) -> bool:
    321        return self.kind == ExecutableKind.VALUE
    322
    323
    324def serialize_env(env: t.Dict[str, t.Any], path: Path) -> t.Dict[str, Executable]:
    325    """Serializes a python function into a self contained dictionary.
    326
    327    Recursively walks a function's globals to store all other references inside of env.
    328
    329    Args:
    330        env: Dictionary to store the env.
    331        path: The root path to seralize. Other modules will not be walked and treated as imports.
    332    """
    333    serialized = {}
    334
    335    for k, v in env.items():
    336        if callable(v):
    337            name = v.__name__
    338            name = k if name == "<lambda>" else name
    339            file_path = Path(inspect.getfile(v))
    340
    341            if _is_relative_to(file_path, path):
    342                serialized[k] = Executable(
    343                    name=name,
    344                    payload=normalize_source(v),
    345                    kind=ExecutableKind.DEFINITION,
    346                    path=str(file_path.relative_to(path.absolute())),
    347                    alias=k if name != k else None,
    348                )
    349            else:
    350                serialized[k] = Executable(
    351                    payload=f"from {v.__module__} import {name}",
    352                    kind=ExecutableKind.IMPORT,
    353                )
    354        elif inspect.ismodule(v):
    355            name = v.__name__
    356            if _is_relative_to(v.__file__, path):
    357                raise SQLMeshError(
    358                    f"Cannot serialize 'import {name}'. Use 'from {name} import ...' instead."
    359                )
    360            postfix = "" if name == k else f" as {k}"
    361            serialized[k] = Executable(
    362                payload=f"import {name}{postfix}",
    363                kind=ExecutableKind.IMPORT,
    364            )
    365        else:
    366            serialized[k] = Executable(payload=repr(v), kind=ExecutableKind.VALUE)
    367
    368    return serialized
    369
    370
    371def prepare_env(
    372    python_env: t.Dict[str, Executable],
    373    env: t.Optional[t.Dict[str, t.Any]] = None,
    374) -> t.Dict[str, t.Any]:
    375    """Prepare a python env by hydrating and executing functions.
    376
    377    The Python ENV is stored in a json serializable format.
    378    Functions and imports are stored as a special data class.
    379
    380    Args:
    381        python_env: The dictionary containing the serialized python environment.
    382        env: The dictionary to execute code in.
    383
    384    Returns:
    385        The prepared environment with hydrated functions.
    386    """
    387    env = {} if env is None else env
    388
    389    for name, executable in sorted(
    390        python_env.items(), key=lambda item: 0 if item[1].is_import else 1
    391    ):
    392        if executable.is_value:
    393            env[name] = ast.literal_eval(executable.payload)
    394        else:
    395            exec(executable.payload, env)
    396            if executable.alias and executable.name:
    397                env[executable.alias] = env[executable.name]
    398    return env
    399
    400
    401def print_exception(
    402    exception: Exception,
    403    python_env: t.Dict[str, Executable],
    404    out: t.TextIO = sys.stderr,
    405) -> None:
    406    """Formats exceptions that occur from evaled code.
    407
    408    Stack traces generated by evaled code lose code context and are difficult to debug.
    409    This intercepts the default stack trace and tries to make it debuggable.
    410
    411    Args:
    412        exception: The exception to print the stack trace for.
    413        python_env: The environment containing stringified python code.
    414        out: The output stream to write to.
    415    """
    416    tb: t.List[str] = []
    417
    418    for error_line in format_exception(exception):
    419        match = re.search(f'File "<string>", line (.*), in (.*)', error_line)
    420
    421        if not match:
    422            tb.append(error_line)
    423            continue
    424
    425        line_num = int(match.group(1))
    426        func = match.group(2)
    427
    428        if func not in python_env:
    429            tb.append(error_line)
    430            continue
    431
    432        executable = python_env[func]
    433        indent = error_line[: match.start()]
    434
    435        error_line = (
    436            f"{indent}File '{executable.path}' (or imported file), line {line_num}, in {func}"
    437        )
    438
    439        code = executable.payload
    440        formatted = []
    441
    442        for i, code_line in enumerate(code.splitlines()):
    443            if i < line_num:
    444                pad = len(code_line) - len(code_line.lstrip())
    445                if i + 1 == line_num:
    446                    formatted.append(f"{code_line[:pad]}{code_line[pad:]}")
    447                else:
    448                    formatted.append(code_line)
    449
    450        tb.extend(
    451            (
    452                error_line,
    453                textwrap.indent(
    454                    os.linesep.join(formatted),
    455                    indent + "  ",
    456                ),
    457                os.linesep,
    458            )
    459        )
    460
    461    out.write(os.linesep.join(tb))
    462
    463
    464def import_python_file(path: Path, relative_base: Path = Path()) -> types.ModuleType:
    465    module_name = str(
    466        path.absolute().relative_to(relative_base.absolute()).with_suffix("")
    467    ).replace(os.path.sep, ".")
    468    # remove the entire module hierarchy in case they were already loaded
    469    parts = module_name.split(".")
    470    for i in range(len(parts)):
    471        sys.modules.pop(".".join(parts[0 : i + 1]), None)
    472
    473    return importlib.import_module(module_name)

def func_globals(func: Callable) -> Dict[str, Any]: View Source

    58def func_globals(func: t.Callable) -> t.Dict[str, t.Any]:
    59    """Finds all global references and closures in a function and nested functions.
    60
    61    This function treats closures as global variables, which could cause problems in the future.
    62
    63    Args:
    64        func: The function to introspect
    65
    66    Returns:
    67        A dictionary of all global references.
    68    """
    69    variables = {}
    70
    71    if hasattr(func, "__code__"):
    72        code = func.__code__
    73
    74        for var in list(_code_globals(code)) + decorators(func):
    75            if var in func.__globals__:
    76                ref = func.__globals__[var]
    77                variables[var] = ref
    78
    79        if func.__closure__:
    80            for var, value in zip(code.co_freevars, func.__closure__):
    81                variables[var] = value.cell_contents
    82
    83    return variables

Finds all global references and closures in a function and nested functions.

This function treats closures as global variables, which could cause problems in the future.

###### Arguments:

- func: The function to introspect

###### Returns:

> A dictionary of all global references.

class ClassFoundException(builtins.Exception): View Source

    86class ClassFoundException(Exception):
    87    pass

Common base class for all non-exit exceptions.

##### Inherited Members

builtins.Exception
  Exception
builtins.BaseException
  with_traceback

def getsource(obj: Any) -> str: View Source

    120def getsource(obj: t.Any) -> str:
    121    """Get the source of a function or class.
    122
    123    inspect.getsource doesn't find decorators in python < 3.9
    124    https://github.com/python/cpython/commit/696136b993e11b37c4f34d729a0375e5ad544ade
    125    """
    126    path = inspect.getsourcefile(obj)
    127    if path:
    128        module = inspect.getmodule(obj, path)
    129
    130        if module:
    131            lines = linecache.getlines(path, module.__dict__)
    132        else:
    133            lines = linecache.getlines(path)
    134
    135        def join_source(lnum: int) -> str:
    136            return "".join(inspect.getblock(lines[lnum:]))
    137
    138        if inspect.isclass(obj):
    139            qualname = obj.__qualname__
    140            source = "".join(lines)
    141            tree = ast.parse(source)
    142            class_finder = _ClassFinder(qualname)
    143            try:
    144                class_finder.visit(tree)
    145            except ClassFoundException as e:
    146                return join_source(e.args[0])
    147        elif inspect.isfunction(obj):
    148            obj = obj.__code__
    149            if hasattr(obj, "co_firstlineno"):
    150                lnum = obj.co_firstlineno - 1
    151                pat = re.compile(r"^(\s*def\s)|(\s*async\s+def\s)|(.*(?<!\w)lambda(:|\s))|^(\s*@)")
    152                while lnum > 0:
    153                    try:
    154                        line = lines[lnum]
    155                    except IndexError:
    156                        raise OSError("lineno is out of bounds")
    157                    if pat.match(line):
    158                        break
    159                    lnum = lnum - 1
    160                return join_source(lnum)
    161    raise SQLMeshError(f"Cannot find source for {obj}")

Get the source of a function or class.

inspect.getsource doesn't find decorators in python < 3.9 https://github.com/python/cpython/commit/696136b993e11b37c4f34d729a0375e5ad544ade

def parse_source(func: Callable) -> ast.Module: View Source

    164def parse_source(func: t.Callable) -> ast.Module:
    165    """Parse a function and returns an ast node."""
    166    return ast.parse(textwrap.dedent(getsource(func)))

Parse a function and returns an ast node.

def decorators(func: Callable) -> List[str]: View Source

    177def decorators(func: t.Callable) -> t.List[str]:
    178    """Finds a list of all the decorators of a callable."""
    179    root_node = parse_source(func)
    180    decorators = []
    181
    182    for node in ast.walk(root_node):
    183        if isinstance(node, (ast.FunctionDef, ast.ClassDef)):
    184            for decorator in node.decorator_list:
    185                name = _decorator_name(decorator)
    186                if name not in IGNORE_DECORATORS:
    187                    decorators.append(name)
    188    return unique(decorators)

Finds a list of all the decorators of a callable.

def normalize_source(obj: Any) -> str: View Source

    191def normalize_source(obj: t.Any) -> str:
    192    """Rewrites an object's source with formatting and doc strings removed by using Python ast.
    193
    194    Args:
    195        obj: The object to fetch source from and convert to a string.
    196
    197    Returns:
    198        A string representation of the normalized function.
    199    """
    200    root_node = parse_source(obj)
    201
    202    for node in ast.walk(root_node):
    203        if isinstance(node, (ast.FunctionDef, ast.ClassDef)):
    204            for decorator in node.decorator_list:
    205                if _decorator_name(decorator) in IGNORE_DECORATORS:
    206                    node.decorator_list.remove(decorator)
    207
    208            # remove docstrings
    209            body = node.body
    210            if body and isinstance(body[0], ast.Expr) and isinstance(body[0].value, ast.Str):
    211                node.body = body[1:]
    212
    213            # remove function return type annotation
    214            if isinstance(node, ast.FunctionDef):
    215                node.returns = None
    216        elif isinstance(node, ast.arg):
    217            node.annotation = None
    218
    219    return to_source(root_node).strip()

Rewrites an object's source with formatting and doc strings removed by using Python ast.

###### Arguments:

- obj: The object to fetch source from and convert to a string.

###### Returns:

> A string representation of the normalized function.

def build_env(obj: Any, *, env: Dict[str, Any], name: str, path: pathlib.Path) -> None: View Source

    222def build_env(
    223    obj: t.Any,
    224    *,
    225    env: t.Dict[str, t.Any],
    226    name: str,
    227    path: Path,
    228) -> None:
    229    """Fills in env dictionary with all globals needed to execute the object.
    230
    231    Recursively traverse classes and functions.
    232
    233    Args:
    234        obj: Any python object.
    235        env: Dictionary to store the env.
    236        name: Name of the object in the env.
    237        path: The module path to serialize. Other modules will not be walked and treated as imports.
    238    """
    239
    240    obj_module = inspect.getmodule(obj)
    241
    242    if obj_module and obj_module.__name__ == "builtins":
    243        return
    244
    245    def walk(obj: t.Any) -> None:
    246        if inspect.isclass(obj):
    247            for decorator in decorators(obj):
    248                if obj_module and decorator in obj_module.__dict__:
    249                    build_env(
    250                        obj_module.__dict__[decorator],
    251                        env=env,
    252                        name=decorator,
    253                        path=path,
    254                    )
    255
    256            for base in obj.__bases__:
    257                build_env(base, env=env, name=base.__qualname__, path=path)
    258
    259            for k, v in obj.__dict__.items():
    260                if k.startswith("__"):
    261                    continue
    262                # traverse methods in a class to find global references
    263                if isinstance(v, (classmethod, staticmethod)):
    264                    v = v.__func__
    265                if callable(v):
    266                    # if the method is a part of the object, walk it
    267                    # else it is a global function and we just store it
    268                    if v.__qualname__.startswith(obj.__qualname__):
    269                        walk(v)
    270                    else:
    271                        build_env(v, env=env, name=v.__name__, path=path)
    272        elif callable(obj):
    273            for k, v in func_globals(obj).items():
    274                build_env(v, env=env, name=k, path=path)
    275
    276    if name not in env:
    277        env[name] = obj
    278        if obj_module and _is_relative_to(obj_module.__file__, path):
    279            walk(obj)
    280    elif env[name] != obj:
    281        raise SQLMeshError(
    282            f"Cannot store {obj} in environment, duplicate definitions found for '{name}'"
    283        )

Fills in env dictionary with all globals needed to execute the object.

Recursively traverse classes and functions.

###### Arguments:

- obj: Any python object.
- env: Dictionary to store the env.
- name: Name of the object in the env.
- path: The module path to serialize. Other modules will not be walked and treated as imports.

class ExecutableKind(builtins.str, enum.Enum): View Source

    286class ExecutableKind(str, Enum):
    287    """The kind of of executable. The order of the members is used when serializing the python model to text."""
    288
    289    IMPORT = "import"
    290    VALUE = "value"
    291    DEFINITION = "definition"
    292    STATEMENT = "statement"
    293
    294    def __lt__(self, other: t.Any) -> bool:
    295        if not isinstance(other, ExecutableKind):
    296            return NotImplemented
    297        values = list(ExecutableKind.__dict__.values())
    298        return values.index(self) < values.index(other)

The kind of of executable. The order of the members is used when serializing the python model to text.

IMPORT = <ExecutableKind.IMPORT: 'import'>
VALUE = <ExecutableKind.VALUE: 'value'>
DEFINITION = <ExecutableKind.DEFINITION: 'definition'>
STATEMENT = <ExecutableKind.STATEMENT: 'statement'>

##### Inherited Members

enum.Enum
  name
  value
builtins.str
  encode
  replace
  split
  rsplit
  join
  capitalize
  casefold
  title
  center
  count
  expandtabs
  find
  partition
  index
  ljust
  lower
  lstrip
  rfind
  rindex
  rjust
  rstrip
  rpartition
  splitlines
  strip
  swapcase
  translate
  upper
  startswith
  endswith
  removeprefix
  removesuffix
  isascii
  islower
  isupper
  istitle
  isspace
  isdecimal
  isdigit
  isnumeric
  isalpha
  isalnum
  isidentifier
  isprintable
  zfill
  format
  format_map
  maketrans

class Executable(sqlmesh.utils.pydantic.PydanticModel): View Source

    301class Executable(PydanticModel):
    302    payload: t.Any
    303    kind: ExecutableKind = ExecutableKind.DEFINITION
    304    name: t.Optional[str] = None
    305    path: t.Optional[str] = None
    306    alias: t.Optional[str] = None
    307
    308    @property
    309    def is_definition(self) -> bool:
    310        return self.kind == ExecutableKind.DEFINITION
    311
    312    @property
    313    def is_import(self) -> bool:
    314        return self.kind == ExecutableKind.IMPORT
    315
    316    @property
    317    def is_statement(self) -> bool:
    318        return self.kind == ExecutableKind.STATEMENT
    319
    320    @property
    321    def is_value(self) -> bool:
    322        return self.kind == ExecutableKind.VALUE

##### Inherited Members

pydantic.main.BaseModel
  BaseModel
  parse_obj
  parse_raw
  parse_file
  from_orm
  construct
  copy
  schema
  schema_json
  validate
  update_forward_refs
sqlmesh.utils.pydantic.PydanticModel
  Config
  dict
  json
  missing_required_fields
  extra_fields
  all_fields
  required_fields

def serialize_env( env: Dict[str, Any], path: pathlib.Path) -> Dict[str, sqlmesh.utils.metaprogramming.Executable]: View Source

    325def serialize_env(env: t.Dict[str, t.Any], path: Path) -> t.Dict[str, Executable]:
    326    """Serializes a python function into a self contained dictionary.
    327
    328    Recursively walks a function's globals to store all other references inside of env.
    329
    330    Args:
    331        env: Dictionary to store the env.
    332        path: The root path to seralize. Other modules will not be walked and treated as imports.
    333    """
    334    serialized = {}
    335
    336    for k, v in env.items():
    337        if callable(v):
    338            name = v.__name__
    339            name = k if name == "<lambda>" else name
    340            file_path = Path(inspect.getfile(v))
    341
    342            if _is_relative_to(file_path, path):
    343                serialized[k] = Executable(
    344                    name=name,
    345                    payload=normalize_source(v),
    346                    kind=ExecutableKind.DEFINITION,
    347                    path=str(file_path.relative_to(path.absolute())),
    348                    alias=k if name != k else None,
    349                )
    350            else:
    351                serialized[k] = Executable(
    352                    payload=f"from {v.__module__} import {name}",
    353                    kind=ExecutableKind.IMPORT,
    354                )
    355        elif inspect.ismodule(v):
    356            name = v.__name__
    357            if _is_relative_to(v.__file__, path):
    358                raise SQLMeshError(
    359                    f"Cannot serialize 'import {name}'. Use 'from {name} import ...' instead."
    360                )
    361            postfix = "" if name == k else f" as {k}"
    362            serialized[k] = Executable(
    363                payload=f"import {name}{postfix}",
    364                kind=ExecutableKind.IMPORT,
    365            )
    366        else:
    367            serialized[k] = Executable(payload=repr(v), kind=ExecutableKind.VALUE)
    368
    369    return serialized

Serializes a python function into a self contained dictionary.

Recursively walks a function's globals to store all other references inside of env.

###### Arguments:

- env: Dictionary to store the env.
- path: The root path to seralize. Other modules will not be walked and treated as imports.

def prepare_env( python_env: Dict[str, sqlmesh.utils.metaprogramming.Executable], env: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: View Source

    372def prepare_env(
    373    python_env: t.Dict[str, Executable],
    374    env: t.Optional[t.Dict[str, t.Any]] = None,
    375) -> t.Dict[str, t.Any]:
    376    """Prepare a python env by hydrating and executing functions.
    377
    378    The Python ENV is stored in a json serializable format.
    379    Functions and imports are stored as a special data class.
    380
    381    Args:
    382        python_env: The dictionary containing the serialized python environment.
    383        env: The dictionary to execute code in.
    384
    385    Returns:
    386        The prepared environment with hydrated functions.
    387    """
    388    env = {} if env is None else env
    389
    390    for name, executable in sorted(
    391        python_env.items(), key=lambda item: 0 if item[1].is_import else 1
    392    ):
    393        if executable.is_value:
    394            env[name] = ast.literal_eval(executable.payload)
    395        else:
    396            exec(executable.payload, env)
    397            if executable.alias and executable.name:
    398                env[executable.alias] = env[executable.name]
    399    return env

Prepare a python env by hydrating and executing functions.

The Python ENV is stored in a json serializable format. Functions and imports are stored as a special data class.

###### Arguments:

- python_env: The dictionary containing the serialized python environment.
- env: The dictionary to execute code in.

###### Returns:

> The prepared environment with hydrated functions.

def print_exception( exception: Exception, python_env: Dict[str, sqlmesh.utils.metaprogramming.Executable], out: <class 'TextIO'> = <_io.TextIOWrapper name='<stderr>' mode='w' encoding='utf-8'>) -> None: View Source

    402def print_exception(
    403    exception: Exception,
    404    python_env: t.Dict[str, Executable],
    405    out: t.TextIO = sys.stderr,
    406) -> None:
    407    """Formats exceptions that occur from evaled code.
    408
    409    Stack traces generated by evaled code lose code context and are difficult to debug.
    410    This intercepts the default stack trace and tries to make it debuggable.
    411
    412    Args:
    413        exception: The exception to print the stack trace for.
    414        python_env: The environment containing stringified python code.
    415        out: The output stream to write to.
    416    """
    417    tb: t.List[str] = []
    418
    419    for error_line in format_exception(exception):
    420        match = re.search(f'File "<string>", line (.*), in (.*)', error_line)
    421
    422        if not match:
    423            tb.append(error_line)
    424            continue
    425
    426        line_num = int(match.group(1))
    427        func = match.group(2)
    428
    429        if func not in python_env:
    430            tb.append(error_line)
    431            continue
    432
    433        executable = python_env[func]
    434        indent = error_line[: match.start()]
    435
    436        error_line = (
    437            f"{indent}File '{executable.path}' (or imported file), line {line_num}, in {func}"
    438        )
    439
    440        code = executable.payload
    441        formatted = []
    442
    443        for i, code_line in enumerate(code.splitlines()):
    444            if i < line_num:
    445                pad = len(code_line) - len(code_line.lstrip())
    446                if i + 1 == line_num:
    447                    formatted.append(f"{code_line[:pad]}{code_line[pad:]}")
    448                else:
    449                    formatted.append(code_line)
    450
    451        tb.extend(
    452            (
    453                error_line,
    454                textwrap.indent(
    455                    os.linesep.join(formatted),
    456                    indent + "  ",
    457                ),
    458                os.linesep,
    459            )
    460        )
    461
    462    out.write(os.linesep.join(tb))

Formats exceptions that occur from evaled code.

Stack traces generated by evaled code lose code context and are difficult to debug. This intercepts the default stack trace and tries to make it debuggable.

###### Arguments:

- exception: The exception to print the stack trace for.
- python_env: The environment containing stringified python code.
- out: The output stream to write to.

def import_python_file( path: pathlib.Path, relative_base: pathlib.Path = PosixPath('.')) -> module: View Source

    465def import_python_file(path: Path, relative_base: Path = Path()) -> types.ModuleType:
    466    module_name = str(
    467        path.absolute().relative_to(relative_base.absolute()).with_suffix("")
    468    ).replace(os.path.sep, ".")
    469    # remove the entire module hierarchy in case they were already loaded
    470    parts = module_name.split(".")
    471    for i in range(len(parts)):
    472        sys.modules.pop(".".join(parts[0 : i + 1]), None)
    473
    474    return importlib.import_module(module_name)