# -*- coding: utf-8 -*- äöü
"""
visaplan.tools.sql: helpers for SQL processing

Autor: Tobias Herp, tobias.herp@visaplan.com

The visaplan.zope.reldb package contains a copy of this module
which generates code for SQLAlchemy.
"""
# Python compatibility:
from __future__ import absolute_import

from six import string_types as six_string_types
from six.moves import map, zip

# Standard library:
from collections import defaultdict

# Local imports:
from visaplan.tools.minifuncs import check_kwargs

__all__ = [# The "fabulous four":
           'insert',
           'update',
           'delete',
           'select',
           # SQL names:
           'check_name',
           "check_alias",
           'replace_names',
           # SQL generation:
           "placeholder",
           "make_transaction_cmd",
           "make_where_mask",
           "make_returning_clause",
           # Formatting:
           "normalize_sql_snippet",
           # helpers:
           'subdict_ne',  # ... generated by:
           'make_dict_extractor',
           # "extract_dict",  # deprecated; please consider using subdict_ne
           'generate_dicts',  # operates on 2-tuple
           "is_sequence",
           # Klassen:
           'SafeNames',
           ]

# Standard library:
from string import digits, letters, uppercase, whitespace

NAMECHARS = frozenset(letters+'._')
ALLNAMECHARS = frozenset(letters+digits+'._')
SNIPPETCHARS = frozenset(uppercase + whitespace)


# ------------------------------------------------- [ SQL basics ... [
def check_name(sqlname, for_select=False):
    """
    Prüfe, ob der übergebene Name einer SQL-Tabelle (oder sonstigen
    benannten Ressource) syntaktisch valide und ohne Quoting
    verwendbar ist.  Gib den Namen im Erfolgsfall unverändert zurück;
    wirf ansonsten einen ValueError.

    >>> check_name('tan.tan')
    'tan.tan'

    Ziffern, die kein Segment einleiten, sind erlaubt:

    >>> check_name('witrabau.p2_witrabau_partners_view')
    'witrabau.p2_witrabau_partners_view'

    Am Anfang eines Segments sind sie aber verboten:

    >>> check_name('witrabau.2_witrabau_partners_view')
    Traceback (most recent call last):
      ...
    ValueError: Error in 'witrabau.2_witrabau_partners_view': '2' must not start a segment

    Doppelte Punkte:
    >>> check_name('tan..tan')
    Traceback (most recent call last):
      ...
    ValueError: Empty segment in 'tan..tan'

    Leere Namen:
    >>> check_name('')
    Traceback (most recent call last):
      ...
    ValueError: Empty segment in ''

    Das Beispiel aus den select-Doctests:
    >>> check_name('evil_field;truncate table users')
    Traceback (most recent call last):
      ...
    ValueError: Invalid chars in 'evil_field;truncate table users': (' ', ';')
    """
    if not isinstance(sqlname, six_string_types):
        raise TypeError('%r is not a string' % (sqlname,))

    invalid = set(sqlname).difference(ALLNAMECHARS)
    if invalid:
        raise ValueError('Invalid chars in %r: %s'
                         % (sqlname, tuple(invalid),))
    for s in sqlname.split('.'):
        if not s:
            raise ValueError('Empty segment in %r' % sqlname)
        if s[0] not in NAMECHARS:
            raise ValueError('Error in %r: %r must not start a segment'
                             % (sqlname, s[0]))
    return sqlname


def check_alias(sqlname):
    """
    Wie check_name, aber mit Unterstützung für AS-Angaben
    (sinnvoll z.B. für returning-Angaben):
    >>> check_alias('id AS user_and_course_id')
    'id AS user_and_course_id'

    Achtung, gleichbedeutend:
    >>> check_alias('id user_and_course_id')
    'id user_and_course_id'

    Offensichtliche Fehler werden erkannt:
    >>> check_alias('id user_and_course_id zwei')
    Traceback (most recent call last):
      ...
    ValueError: Part 'id user_and_course_id zwei': AS expected, found: 'user_and_course_id'
    >>> check_alias('id AS user_and_course_id zwei')
    Traceback (most recent call last):
      ...
    ValueError: Part too long ('id AS user_and_course_id zwei')
    >>> check_alias('id as')
    Traceback (most recent call last):
      ...
    ValueError: Misplaced AS ('id as')
    >>> check_alias('as user_and_course_id ')
    Traceback (most recent call last):
      ...
    ValueError: Misplaced AS ('as user_and_course_id ')
    >>> check_alias('   ')
    Traceback (most recent call last):
      ...
    ValueError: Empty name or alias ('   ')
    """
    if sqlname == '*':
        return sqlname
    words = sqlname.split()
    if not words:
        raise ValueError('Empty name or alias (%(sqlname)r)' % locals())
    if words[3:]:
        raise ValueError('Part too long (%(sqlname)r)' % locals())
    if not words[2:]:
        for word in words:
            if word.upper() == 'AS':
                raise ValueError('Misplaced AS (%(sqlname)r)' % locals())
            check_name(word)
    else:
        if words[1].lower() != 'as':
            raise ValueError('Part %r: AS expected, found: %r'
                             % (sqlname, words[1]))
        for word in words[0::2]: # 0, 2, 4 ... aber: siehe oben
            if word.upper() == 'AS':
                raise ValueError('Misplaced AS (%(sqlname)r)' % locals())
            check_name(word)
    return sqlname


# ------------------------------- [ Placeholders convention ... [
def pg_placeholder(key):
    """
    Turn a field name in a psycopg2-style placeholder

    >>> pg_placeholder('name')
    '%(name)s'

    This function is usually not directly used; it is used internally by the
    higher-level functions like make_where_mask
    which take care of checking the key for dangerous values (to prevent SQL injection).

    This is the placeholder factory used in this module;
    there is currently no facility to create SQL code generators which
    use another placeholder convention, e.g. for SQLAlchemy.
    """
    if not key:
        raise ValueError('Empty field name %(key)r!' % locals())
    return key.join(('%(', ')s'))


def sqlalchemy_placeholder(key):
    """
    Turn a field name in an SQLAlchemy-style placeholder

    >>> sqlalchemy_placeholder('name')
    ':name'

    This is the placeholder factory to generate SQL code like expected by SQLAlchemy;
    it is used in the copy of this module in the visaplan.zope.reldb package.
    """
    if not key:
        raise ValueError('Empty field name %(key)r!' % locals())
    return ':'+key

placeholder = pg_placeholder
# ------------------------------- ] ... Placeholders convention ]
# ------------------------------------------------- ] ... SQL basics ]


def generate_dicts(sqlres, names=None):
    """
    Generate the rows from a 2-tuple like returned by db.query
    as dictionaries.

    Arguments:

      sqlres - a 2-tuple; the first item is a list of dicts (one per returned field),
               the second is a list of row value tuples.
      names -- DEPRECATED
               The given names MUST match the field names
               extracted from the dictionaries in sqlres[0];
               otherwise we would put the values in the wrong places.

    We'll create a little factory function for our test result tuples:
    >>> def MockResult(keys, rows):
    ...     info = []
    ...     for key in keys.split():
    ...         info.append({'scale': None, 'name': key, 'precision': None, 'width': None, 'null': None, 'type': 'n'})
    ...     return (info, list([tuple(row) for row in rows]))
    >>> res = MockResult('id', ([3],))
    >>> res
    ([{'scale': None, 'name': 'id', 'precision': None, 'width': None, 'null': None, 'type': 'n'}], [(3,)])
    >>> generate_dicts(res)  # doctest: +ELLIPSIS
    <generator object generate_dicts at ...>
    >>> list(generate_dicts(res))
    [{'id': 3}]
    >>> list(generate_dicts(res, names=('id',)))
    [{'id': 3}]
    >>> list(generate_dicts(res, names='id'))
    [{'id': 3}]
    >>> list(generate_dicts(res, names='*'))
    [{'id': 3}]

    >>> res2 = MockResult('id status', ([1, 'ok'], [2, 'ok'], [5, 'oops']))

    We'll use a little test helper to have a portable test:
    >>> def lotd(seq):  # list of tupelized dicts
    ...     res = []
    ...     for dic in seq:
    ...         res.append(sorted(dic.items()))
    ...     return res
    >>> lotd(generate_dicts(res2))
    [[('id', 1), ('status', 'ok')], [('id', 2), ('status', 'ok')], [('id', 5), ('status', 'oops')]]

    With the number of fields reduced to one, we don't need this test helper:
    >>> list(generate_dicts(res2, ['id']))
    [{'id': 1}, {'id': 2}, {'id': 5}]

    However, the field names must match the fields contained in the ResultProxy:
    >>> list(generate_dicts(res2, ['status']))
    Traceback (most recent call last):
      ...
    ValueError: The given names ['status'] don't match the first 1 field name(s) returned (['id'])

    """
    detected = [topic['name'] for topic in sqlres[0]]
    if names is None or names == '*':
        names = detected
    else:
        if not is_sequence(names):
            names = names.split()
        elif not isinstance(names, list):
            names = list(names)
        checked = len(names)
        expected = detected[:checked]
        if names != expected:
            raise ValueError("The given names %(names)r don't match the "
                             'first %(checked)d field name(s) returned '
                             '(%(expected)r)'
                             % locals())
    raw = sqlres[1]
    for row in raw:  # no dict(list(zip())) necessary, right?
        yield dict(zip(names, row))


def normalize_sql_snippet(snippet):
    """
    Normalisiere einen SQL-Schnipsel und gib ihn zurück.
    Im Ergebnis sind nur Großbuchstaben und ggf. Leerzeichen enthalten;
    evtl. wird ein ValueError geworfen.

    Achtung: keine Namen (Identifier), keine Werte!

    >>> normalize_sql_snippet('  read  only  ')
    'READ ONLY'

    >>> normalize_sql_snippet('select * from my.table;')
    Traceback (most recent call last):
      ...
    ValueError: Invalid chars in 'select * from my.table;': ('*', '.', ';')
    """
    res = ' '.join(snippet.upper().split())
    invalid = set(res).difference(SNIPPETCHARS)
    if invalid:
        raise ValueError('Invalid chars in %r: %s'
                         % (snippet, tuple(sorted(invalid)),))
    return res

ISOLATION_LEVELS = set(['SERIALIZABLE',
                        'REPEATABLE READ',
                        'READ COMMITTED',
                        'READ UNCOMMITTED', # in PostgreSQL wie read committed
                        ])
TRANSACTION_MODES = set(['READ WRITE',
                         'READ ONLY',
                         ])
for item in ISOLATION_LEVELS:
    TRANSACTION_MODES.add(item)
    TRANSACTION_MODES.add('ISOLATION LEVEL '+item)


# http://www.postgresql.org/docs/9.1/static/sql-start-transaction.html
# http://www.postgresql.org/docs/9.1/static/sql-set-transaction.html
def make_transaction_cmd(action, *args):
    """
    Nimm einige Spezifikationen für Transaktionen entgegen
    und erzeuge daraus das entsprechende SQL-Statement.

    >>> make_transaction_cmd('BEGIN')
    'BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;'
    >>> make_transaction_cmd('SET', 'read only')
    'SET TRANSACTION READ ONLY;'
    """
    assert action in ('BEGIN', 'SET')
    isolation_level = None
    other_specs = []
    for a in args:
        AA = normalize_sql_snippet(a)
        if AA in ISOLATION_LEVELS:
            isolation_level = 'ISOLATION LEVEL ' + AA
        elif AA in TRANSACTION_MODES:
            if AA.startswith('ISOLATION LEVEL '):
                isolation_level = AA
            else:
                other_specs.append(AA)
        else:
            raise ValueError('Unbekannter Transaktionsmodus: %r' % (AA,))
    if isolation_level is None and action == 'BEGIN':
        isolation_level = 'ISOLATION LEVEL READ COMMITTED'
    if isolation_level is not None:
        other_specs.insert(0, isolation_level)
    return '%s TRANSACTION %s;' % (action, ', '.join(other_specs))


class SafeNames(dict):
    """
    Für die Generierung von SQL-Anweisungen:
    Ein Dictionary, das die ihm bekannten Werte ersetzt
    und für die anderen den Platzhalter repliziert.

    >>> smartie = SafeNames(foo='bar')
    >>> '%(foo)s %(baz)s' % smartie
    'bar %(baz)s'
    """
    def __getitem__(self, key):
        """
        >>> sd = SafeNames()
        >>> sd.__getitem__('missing')
        '%(missing)s'
        """
        try:
            return dict.__getitem__(self, key)
        except KeyError:
            check_name(key)
            return placeholder(key)


def replace_names(sql, **kwargs):
    """
    Zur Vorverarbeitung: Sicheres Ersetzen von Tabellen- und
    sonstigen Namen, bevor der Datenbankadapter für das Quoting der
    Werte sorgt.

    Die Tabellen...namen werden *nicht* gequotet, weil das das Ende
    der Groß-/Kleinschreibungstoleranz bedeuten würde; stattdessen
    wird sichergestellt, daß keine gefährlichen Zeichen enthalten
    sind.

    sql - das SQL-Statement
    kwargs -- Platzhalter und Werte für die Namen von Tabellen o.ä.
              (werden mit check_name überprüft)

    >>> replace_names('SELECT * FROM %(table)s WHERE val=%(val)s;', table='fozzie')
    'SELECT * FROM fozzie WHERE val=%(val)s;'

    For invalid values we get a ValueError:
    >>> replace_names('SELECT * FROM %(table)s;', table='honk;drop database')
    Traceback (most recent call last):
      ...
    ValueError: Invalid chars in 'honk;drop database': (' ', ';')
    """
    for v in kwargs.values():
        check_name(v)
    dic = SafeNames(kwargs)
    return sql % dic


WHERE = intern('WHERE')
def make_where_mask(dic, fields=None, keyword=WHERE):
    """
    Komfort-Funktion; wenn die Query-Daten schon als dict vorliegen,
    braucht man sich die WHERE-Bedingung nicht aus den Fingern zu saugen.
    Ohne Daten ist das Ergebnis ein Leerstring (also auch ohne 'WHERE').

    dic -- die Query-Daten. Es werden nur die Schlüssel verwendet; die Werte
           werden beim Absenden der Query vom SQL-Adapter eingesetzt.

    fields -- wenn angegeben, werden die in <fields> aufgeführten Schlüssel
              bevorzugt behandelt; ansonsten bestimmt sich die Reihenfolge
              nach ASCII-Sortierung.

    >>> make_where_mask({})
    ''
    >>> make_where_mask({'answer': 42})
    'WHERE answer = %(answer)s'
    >>> query_data={'zwei': 2, 'eins': 1}
    >>> make_where_mask(query_data)
    'WHERE eins = %(eins)s AND zwei = %(zwei)s'
    >>> len(query_data)
    2
    >>> make_where_mask(query_data, ['zwei', 'eins'])
    'WHERE zwei = %(zwei)s AND eins = %(eins)s'

    'drei' ist in der fields-Liste nicht erwähnt und kommt daher zum Schluß:

    >>> query_data['drei'] = 3
    >>> make_where_mask(query_data, ['zwei', 'eins'])
    'WHERE zwei = %(zwei)s AND eins = %(eins)s AND drei = %(drei)s'

    Unterstützung von Sequenzen:

    >>> make_where_mask({'status': ['new', 'reserved']})
    'WHERE status = ANY(%(status)s)'

    Bei Verwendung von Gruppierung und Aggregatfunktionen:

    >>> make_where_mask({'status': ['new', 'reserved']}, keyword='HAVING')
    'HAVING status = ANY(%(status)s)'
    """
    assert keyword in (WHERE, 'HAVING')
    keys = sorted(check_name(key) for key in dic.keys())
    if fields:
        po = len(fields)
        tmp = []
        for key in keys:
            try:
                idx = fields.index(key)
                tmp.append((idx, key))
            except ValueError:
                tmp.append((po, key))
                po += 1
        tmp.sort()
        keys = [tup[1] for tup in tmp]
    if keys:
        res = []
        for key in keys:
            if is_sequence(dic[key]):
                res.append(''.join((key, ' = ANY(',
                                    placeholder(key),
                                    ')')))
            else:
                res.append(''.join((key, ' = ',
                                    placeholder(key))))
        return ' '.join((keyword, ' AND '.join(res)))
    return ''


def is_sequence(arg):
    """
    Handelt es sich im Sinne von SQL um eine Sequenz (abzugleichen mit
    ... = ANY (...) anstelle von ... = ...)?

    >>> is_sequence('test')
    False
    >>> is_sequence(['a', 'b'])
    True
    >>> is_sequence(42)
    False

    Auch Generatoren werden erkannt:
    >>> is_sequence(xrange(1, 3))
    True
    """
    if hasattr(arg, 'strip'):
        return False
    return (hasattr(arg, "__getitem__") or
            hasattr(arg, "__iter__")
            )


def make_returning_clause(fields):
    """
    Gib eine RETURNING-Klausel zurück (PostgreSQL-Erweiterung gegenüber
    dem SQL-Standard, ab Version 9.1):

    >>> make_returning_clause('*')
    'RETURNING *'
    >>> make_returning_clause('id')
    'RETURNING id'
    >>> make_returning_clause(['tan', 'status'])
    'RETURNING tan, status'

    Solche returning-Ausdrücke werden vom Datenbanktreiber derzeit leider nicht
    korrekt interpretiert - es kommt ein Feldname "is AS some_better_name"
    dabei heraus.  Deshalb prüfen wir hier vorerst weiterhin mit check_name,
    nicht mit check_alias:
    >>> make_returning_clause('id AS some_better_name')
    Traceback (most recent call last):
        ...
    ValueError: Invalid chars in 'id AS some_better_name': (' ',)
    """
    if fields == '*':
        return 'RETURNING *'
    if not is_sequence(fields):
        liz = [fields]
    else:
        liz = fields
    return 'RETURNING ' + ', '.join(map(check_name, liz))


def extract_dict(fields, source, pop=1, noempty=1):
    """
    Extrahiere die angegebenen Felder aus dem Quell-Dictionary,
    um sie beispielsweise in eine andere Tabelle zu schreiben.
    Dabei werden die gefundenen Schlüssel normalerweise aus der Quelle
    gelöscht:

    >>> source={'tan': 123, 'status': 'new', 'owner_id': 'Willy'}
    >>> extract_dict(['status'], source)
    {'status': 'new'}
    >>> source
    {'tan': 123, 'owner_id': 'Willy'}

    Wird pop=0 übergeben, bleibt die Quelle unverändert erhalten:

    >>> extract_dict(['owner_id'], source, pop=0)
    {'owner_id': 'Willy'}
    >>> source
    {'tan': 123, 'owner_id': 'Willy'}

    In der Quelle nicht vorhandene Schlüssel werden übergangen,
    erzeugen also keinen Fehler:

    >>> extract_dict(['owner_id', 'group_id'], source, pop=0)
    {'owner_id': 'Willy'}
    >>> source
    {'tan': 123, 'owner_id': 'Willy'}

    Mit noempty=True (Standardwert) werden "leere" Werte (wie nach dem
    Absenden von Web-Formularen sehr häufig) weggelassen:

    >>> form={'group_id': 'group_abc', 'status': '', 'zahl': '0'}
    >>> all=['group_id', 'status', 'zahl']
    >>> extract_dict(all, form, pop=1, noempty=1)
    {'group_id': 'group_abc', 'zahl': '0'}

    Diese leeren Werte sind im Falle von pop=True dann trotzdem in der
    Quelle gelöscht:
    >>> form
    {}

    Siehe auch die allgemeinere Funktion visaplan.tools.dicts.subdict
    und ihren Wrapper subdict_forquery.
    """
    get = pop and source.pop or source.get
    res = {}
    if noempty:
        for field in fields:
            if field in source:
                val = get(field)
                if val not in (None, ''):
                    res[field] = val
    else:
        for field in fields:
            if field in source:
                res[field] = get(field)
    return res


def _make_empty_detector(empty_values, falsy_lists_considered_empty):
    """
    Create a helper for make_dict_extractor().

    >>> _med = _make_empty_detector

    Normally we'll consider a common set of simple values empty,
    and empty lists etc. as well:

    >>> isempty = _med(set([None, '']), 1)
    >>> isempty(0)
    False
    >>> isempty(None)
    True
    >>> isempty('')
    True
    >>> isempty([])
    True
    >>> isempty([''])
    False
    >>> isempty(())
    True
    >>> isempty({})
    True
    >>> isempty(set())
    True
    >>> isempty(frozenset())
    True

    We might want to disregard empty sequences:
    >>> ie2 = _med(set([None, '']), 0)
    >>> ie2(0)
    False
    >>> ie2(None)
    True
    >>> ie2('')
    True
    >>> ie2(())
    False
    >>> ie2(set())
    0
    >>> ie2(frozenset())
    0

    In this case, however, the call will fail for unhashable values:
    >>> ie2([])
    Traceback (most recent call last):
      ...
    TypeError: unhashable type: 'list'
    >>> ie2([''])
    Traceback (most recent call last):
      ...
    TypeError: unhashable type: 'list'
    >>> ie2({})
    Traceback (most recent call last):
      ...
    TypeError: unhashable type: 'dict'

    """
    def consider_both(val):
        if isinstance(val, (list, tuple, dict, set, frozenset)):
            return not val
        return val in empty_values

    def consider_fixed_only(val):
        """
        Currently will fail with a TypeError for unhashable values
        """
        return val in empty_values

    def consider_lists_only(val):
        if isinstance(val, (list, tuple, dict, set, frozenset)):
            return not val
        return False

    if empty_values:
        return (consider_both if falsy_lists_considered_empty
                else consider_fixed_only)
    else:
        return consider_lists_only


def make_dict_extractor(**kwargs):
    """
    Create a subdict extractor for SQL use
    which will ...

    - automatically drop fields with "empty" values
      (which are often useless for filtering),
      unless `noempty` is `False`
    - gracefully *ignore* any missing fields
    - never use any default values

    In some of these respects the generated function differs from
    .dicts.subdict.

    >>> def tst(func, *args, **kwargs):
    ...     return sorted(func(*args, **kwargs).items())
    >>> fne = make_dict_extractor()
    >>> tst(fne, {'number': 0, 'str': '', 'anything': None})
    [('number', 0)]

    Now we'll create an extrator which won't check for "empty" values:
    >>> fie = make_dict_extractor(noempty=False)

    >>> tst(fie, {'number': 0, 'str': '', 'anything': None})
    [('anything', None), ('number', 0), ('str', '')]

    If we are not interested in *all* keys of the dictionary
    we can specify the fields to use; if we use a string, it will be split:

    >>> tst(fie, {'number': 2, 'str': '', 'anything': None}, 'number missing')
    [('number', 2)]

    We publish a standard extractor function `subdict_ne`
    which regards the default `empty_values`:

    >>> subdict_ne({'number': 0, 'str': '', 'anything': None})
    {'number': 0}

    Other than `.dicts.subdict`, this function won't ever complain
    about missing keys nor inject default values:

    >>> subdict_ne({'number': 42, 'anything': None}, ['number', 'missing'])
    {'number': 42}

    Currently, falsy lists etc. are always considered empty:
    >>> subdict_ne({'status': [],
    ...             'anything': None}, ['status', 'missing'])
    {}

    >>> subdict_ne({'status': ['new', 'reserved'],
    ...             'anything': None}, ['status', 'missing'])
    {'status': ['new', 'reserved']}

    See as well `.dicts.subdict_forquery` which sports *all* options of the `subdict` function
    (which it calls internally) but currently only filters out `None` values.

    TODO: add a `strict` option here as well; but first we'll need to consider
          whether it should be True or False by default.
    """
    do_pop = kwargs.pop('pop', 1) or 0
    if 'noempty' in kwargs:
        noempty = kwargs.pop('noempty')
        empty_values = kwargs.pop('empty_values', set([None, '']))
        falsy_empty = kwargs.pop('falsy_lists_considered_empty', noempty)
    else:
        empty_values = kwargs.pop('empty_values', set([None, '']))
        if empty_values:
            falsy_empty = kwargs.pop('falsy_lists_considered_empty', 1)
            noempty = 1
        else:
            falsy_empty = kwargs.pop('falsy_lists_considered_empty', 0)
            noempty = falsy_empty

    def sql_subdict(source, fields=None, pop=do_pop):
        get = pop and source.pop or source.get
        res = {}
        if fields is None:
            fields = source.keys()  # we iterate this once only
        elif isinstance(fields, six_string_types):
            fields = fields.split()
        for field in fields:
            if field in source:
                res[field] = get(field)
        return res

    if not noempty:
        return sql_subdict

    isempty = _make_empty_detector(empty_values, falsy_empty)

    def sql_subdict_noempty(source, fields=None, pop=do_pop):
        get = pop and source.pop or source.get
        res = {}
        if fields is None:
            fields = source.keys()  # we iterate this once only
        elif isinstance(fields, six_string_types):
            fields = fields.split()
        for field in fields:
            if field in source:
                val = get(field)
                if not isempty(val):
                    res[field] = val
        return res

    return (sql_subdict_noempty if noempty
            else sql_subdict)
subdict_ne = make_dict_extractor()


# ----------------------------------------- [ options extraction ... [
def _fields_and_querydata(args, kwargs, qd_name='query_data'):  # - [[
    """
    Helper function for functions which liberally accept a `fields` and a
    `query_data` argument, in (almost) any order; see

    - select()

    We return a 2-tuple for the `fields` and the `query_data` argument;
    the fields argument is expanded to '*', if None:

    >>> _fields_and_querydata([], {})
    ('*', None)

    >>> _fields_and_querydata(['id status'], {})
    ('id, status', None)

    >>> _fields_and_querydata([{'table': 'any_table'}],
    ...                       {'names': {'table': 'other_table'}},
    ...                       'names')
    Traceback (most recent call last):
      ...
    TypeError: Duplicate names spec. ({'table': 'any_table'}; we already have {'table': 'other_table'})

    """
    have_fields = 'fields' in kwargs
    have_qdata = qd_name in kwargs
    pop = kwargs.pop
    if have_fields:
        fields = pop('fields')
    else:
        fields = None
    if have_qdata:
        query_data = pop(qd_name)
    else:
        query_data = None
    for a in args:
        if isinstance(a, dict):
            if have_qdata:
                raise TypeError('Duplicate %(qd_name)s spec. (%(a)r; '
                                'we already have %(query_data)r)'
                                % locals())
            query_data = a
            have_qdata = True
        elif isinstance(a, six_string_types) or is_sequence(a):
            if have_fields:
                raise TypeError('Duplicate fields spec. (%(a)r; '
                                'we already have %(fields)r)'
                                % locals())
            fields = a
            have_fields = a
        elif have_qdata and have_fields:
            raise TypeError('Surplus anonymous argument %(a)r'
                            % locals())
        elif a is None:
            if not have_fields:
                fields = a
                have_fields = True
            else:
                assert not have_qdata, "Yes, I'm sure!"
                query_data = a
                have_qdata = True
        else:
            raise ValueError("Don't know where to put %(a)r" % locals())

    if fields is None:
        fields = '*'
    elif fields == '*':
        pass
    elif fields:
        if isinstance(fields, six_string_types):
            fields = fields.split()
        fields = ', '.join(check_name(field) for field in fields)
    else:
        fields = '*'

    return fields, query_data  # ------- ] ... _fields_and_querydata ]


def _some_dicts(args, kwargs, names=None):  # ---- [ _some_dicts ... [
    """
    Take some dict arguments from the names arguments (kwargs)
    or the anonymous arguments (args) of the calling function,
    and return a tuple. The kwargs are consumed;
    the args are considered to contain nothing else.

    >>> _some_dicts([{'table': 'the_table'}, {'id': 42}], {})
    ({'table': 'the_table'}, {'id': 42})

    >>> _some_dicts([{'table': 'the_table'}], {})
    ({'table': 'the_table'}, None)
    """
    if names is None:
        names = ['names', 'query_data']
    elif not is_sequence(names):
        names = names.split()
    elif not isinstance(names, tuple):
        names = tuple(names)
    found = defaultdict(lambda: None)
    have = {}
    pop = kwargs.pop
    for name in names:
        if name in kwargs:
            have[name] = True
            found[name] = pop(name)
        else:
            have[name] = False
    for a in args:
        if a is not None and not isinstance(a, dict):
            raise ValueError('Looking for dicts here; found %(a)r'
                             % locals())
        free = None
        for name in names:
            if not have[name]:
                free = name
                break
        if free is None:
            raise ValueError("Don't know where to put %(a)r" % locals())
        found[free] = a
        have[free] = True
    return tuple(found[name] for name in names)  # ] ... _some_dicts ]
# ----------------------------------------- ] ... options extraction ]


# ---------------------------------- [ SQL-Statements generieren ... [
def insert(table, dict_of_values,  # ------------- [ insert ... [
           **kwargs):
    """
    Return a simple "INSERT INTO ..." SQL statement with placeholders
    and a values dict.

    Mandatory arguments:

      table -- name of the table (may contain the schema)
      dict_of_values -- a Python dict containing the values

    Keyword-only options:

      returning -- like supported by PostgreSQL 9.1+
      strict -- if False, unknown keyword arguments will be silently ignored;
                if True (default), a TypeError will be raised.

    >>> insert('die_tabelle', {'name': 'Zaphod', 'heads': 2})[0]
    'INSERT INTO die_tabelle (heads, name) VALUES (%(heads)s, %(name)s);'
    >>> insert('die_tabelle', {'name': 'Zaphod', 'heads': 2},
    ...        returning='id')[0]
    'INSERT INTO die_tabelle (heads, name) VALUES (%(heads)s, %(name)s) RETURNING id;'

    (As for the INSERT statement, the 2nd returned value is not very interesting -
    it is the the `dict_of_values` which was given as an argument.)

    Table names are checked for invalid values:
    >>> insert('evil.table;truncate table users', {'status': 'done'})[0]
    Traceback (most recent call last):
    ...
    ValueError: Invalid chars in 'evil.table;truncate table users': (' ', ';')

    >>> insert('another.table', {'evil_field;drop database': 42})
    Traceback (most recent call last):
    ...
    ValueError: Invalid chars in 'evil_field;drop database': (' ', ';')

    For unknown keyword arguments, we get a TypeError:
    >>> insert('the_guide', {'name': 'Earth'}, mostly='harmless')
    Traceback (most recent call last):
      ...
    TypeError: Unknown option 'mostly' found!

    """
    keys = list(check_name(key) for key in dict_of_values.keys())
    rows = ', '.join(keys)
    values = ', '.join([placeholder(key)
                        for key in keys
                        ])
    query_l = [replace_names('INSERT INTO %(table)s',
                             table=table),
               '(%s)' % rows,
               'VALUES (%s)' % values,
               ]
    check_kwargs(kwargs,
                 allowed=set(['returning', 'strict']))
    returning = kwargs.get('returning')
    if returning:
        query_l.append(make_returning_clause(returning))
    statements = [' '.join(query_l)]
    statements.append('')
    query = ';'.join(statements)
    return query, dict_of_values
    # -------------------------------------------- ] ... insert ]


def update(table, dict_of_values,  # ------------- [ update ... [
           query_data=None,
           **kwargs):
    """
    Return a simple "UPDATE ..." SQL statement with placeholders
    and a values dict.

    Arguments:

      table -- name of the table (may contain the schema)
      dict_of_values -- a Python dict containing the new values
      query_data -- another dict with a disjunct set of keys
                    to restrict the rows changed.

    Keyword-only options:

      where -- a "WHERE ..." string (which may contain placeholders);
               currently necessary if some fields are both changed
               and used in the filtering `query_data`
      returning -- like supported by PostgreSQL 9.1+
      strict -- if False, unknown keyword arguments will be silently ignored;
                if True (default), a TypeError will be raised.
      fork -- Rarely specified: Whether to create a copy of the `query_data`
              before merging in the `dict_of_values`.
              Defaults to True.

      fork -- wenn <query_data> nach dem Methodenaufruf noch verwendet
              werden soll, muß intern eine Kopie angelegt werden

    >>> tup = update('the_table', {'status': 'done'},
    ...              {'id': [47, 11]})
    >>> tup[0]
    'UPDATE the_table SET status=%(status)s WHERE id = ANY(%(id)s);'
    >>> tup[1]
    {'status': 'done', 'id': [47, 11]}

    Die Funktion akzeptiert zwei Dictionary-Optionen, die oft zusammen verwendet werden;
    was passiert mit etwaigen Variablen?

    >>> values = {'status': 'success'}
    >>> query_data = {'id': [1, 2, 5]}
    >>> tup2 = update('the.table', values, query_data)
    >>> tup2[0]
    'UPDATE the.table SET status=%(status)s WHERE id = ANY(%(id)s);'
    >>> tup2[1]
    {'status': 'success', 'id': [1, 2, 5]}

    Beide Eingabe-Dictionarys sind unverändert:

    >>> values
    {'status': 'success'}
    >>> query_data
    {'id': [1, 2, 5]}

    Was ist, wenn ein Feldname sowohl im Filter als auch in den zugewiesenen Werten auftaucht?

    >>> update('the_table', {'status': 'done'}, {'status': None})
    Traceback (most recent call last):
    ...
    ValueError: key 'status' is both in query data (None) and update data ('done')!

    In solchen Fällen kann man sich helfen, indem man selbst einen where-String angibt:

    >>> tup3 = update('the_table', {'status': 'done'},
    ...               where='WHERE status is NULL')
    >>> tup3[0]
    'UPDATE the_table SET status=%(status)s WHERE status is NULL;'
    >>> tup3[1]
    {'status': 'done'}

    >>> tup4 = update('the_table', {'status': 'done'}, {'status_old': 'in_progress'},
    ...               where='WHERE status = %(status_old)s')
    >>> tup4[0]
    'UPDATE the_table SET status=%(status)s WHERE status = %(status_old)s;'
    >>> tup4[1]
    {'status': 'done', 'status_old': 'in_progress'}

    Table names are checked for invalid values:
    >>> update('evil.table;truncate table users', {'status': 'done'})[0]
    Traceback (most recent call last):
    ...
    ValueError: Invalid chars in 'evil.table;truncate table users': (' ', ';')

    >>> update('another.table', {'evil_field;drop database': 42})
    Traceback (most recent call last):
    ...
    ValueError: Invalid chars in 'evil_field;drop database': (' ', ';')

    The 2nd part of the returned 2-tuple will always be a dictionary,
    even if no query_data had been given:
    >>> update('the_table', {'status': 'done'})
    ('UPDATE the_table SET status=%(status)s;', {'status': 'done'})

    (Well, as for the update function, this is not very surprising,
    since a call to it *without* any dictionary data doesn't make any sense.)

    For unknown keyword arguments, we get a TypeError:
    >>> update('the_table', {'status': 'done'}, unknown=42)
    Traceback (most recent call last):
    ...
    TypeError: Unknown option 'unknown' found!
    """
    keys = list(check_name(key) for key in dict_of_values.keys())
    qset = ', '.join([key + '=' + placeholder(key)
                      for key in keys
                      ])
    query_l = [replace_names('UPDATE %(table)s SET',
                             table=table),
               qset,
               ]
    if query_data is not None:
        if not isinstance(query_data, dict):
            raise ValueError('query_data must be a dict; found %(query_data)r'
                             % locals())
        query_keys = set(query_data.keys())
        value_keys = set(keys)
        keys_of_both = value_keys.intersection(query_keys)
        if keys_of_both:
            # Löschen aus Set während Iteration nicht erlaubt;
            # also iteration über "Kopie":
            for key in sorted(keys_of_both):
                u_val = dict_of_values[key]
                q_val = query_data[key]
                if u_val == q_val:
                    del dict_of_values[key]
                    keys_of_both.remove(key)
                else:
                    raise ValueError ('key %(key)r is both'
                                 ' in query data (%(q_val)r)'
                                 ' and update data (%(u_val)r)!'
                                 % locals())
        if not dict_of_values:
            raise ValueError('Empty update data!')
        if keys_of_both:
            raise ValueError('intersection of value keys and '
                             'query keys (%(keys_of_both)s: '
                             'currently unsupported!'
                             % locals())
        fork = kwargs.get('fork', True)
        if fork:
            query_data = dict(query_data)  # wg. Wiederverwendung!
    else:
        query_data = {}
        fork = kwargs.get('fork', False)
    check_kwargs(kwargs,
                 allowed=set(['returning', 'where', 'fork',
                              'strict']))
    where = kwargs.get('where')
    if where and not isinstance(where, six_string_types):
        raise ValueError('where must be a string; found %(where)r'
                         % locals())
    if query_data and not where:
        where = make_where_mask(query_data)

    if where:
        query_l.append(where)

    returning = kwargs.get('returning')
    if returning:
        query_l.append(make_returning_clause(returning))
    queries = [' '.join(query_l)+';']
    query = ''.join(queries)
    # nicht alle "Query-Daten" dienen der Filterung (siehe oben, keys_of_both)
    query_data.update(dict_of_values)
    return query, query_data
    # -------------------------------------------- ] ... update ]


def delete(table,  # ----------------------------- [ delete ... [
           query_data=None,
           **kwargs):
    """
    Return a simple "DELETE ..." SQL statement with placeholders
    and a values dict.

    Arguments:

      table -- name of the table (may contain the schema)
      query_data -- a dict with a disjunct set of keys
                    to restrict the rows deleted.

    Keyword-only options:

      where -- a "WHERE ..." string (which may contain placeholders)
      returning -- like supported by PostgreSQL 9.1+
      strict -- if False, unknown keyword arguments will be silently ignored;
                if True (default), a TypeError will be raised.

    Caution: Without a WHERE criterion (whether specified as `where` or `query_data` option,
             or a combination of both), the whole table will be deleted (like TRUNCATE, but slower and
             without the need for the TRUNCATE permission granted);
             there is currently nothing to prevent his!

    >>> tup1 = delete('the_table', {'status': 'done'})
    >>> tup1[0]
    'DELETE FROM the_table WHERE status = %(status)s;'
    >>> tup1[1]
    {'status': 'done'}

    SQL injection prevention:
    >>> delete('evil.table;drop database')
    Traceback (most recent call last):
    ...
    ValueError: Invalid chars in 'evil.table;drop database': (' ', ';')

    >>> delete('another.table', {'evil_field;drop database': 42})
    Traceback (most recent call last):
    ...
    ValueError: Invalid chars in 'evil_field;drop database': (' ', ';')

    Table names are checked for invalid values:
    >>> delete('evil.table;truncate table users', {'status': 'done'})[0]
    Traceback (most recent call last):
    ...
    ValueError: Invalid chars in 'evil.table;truncate table users': (' ', ';')

    The 2nd part of the returned 2-tuple will always be a dictionary,
    even if no query_data had been given:
    >>> delete('slowly_truncated.table')
    ('DELETE FROM slowly_truncated.table;', {})

    For unknown keyword arguments, we get a TypeError:
    >>> delete('fancy.table', unknown=42)
    Traceback (most recent call last):
    ...
    TypeError: Unknown option 'unknown' found!
    """
    query_l = [replace_names('DELETE FROM %(table)s',
                             table=table),
               ]
    where = kwargs.get('where')
    if where and not isinstance(where, six_string_types):
        raise ValueError('where must be a string; found %(where)r'
                         % locals())
    if query_data and not where:
        where = make_where_mask(query_data)
    if where:
        query_l.append(where)
    returning = kwargs.get('returning')
    check_kwargs(kwargs,
                 allowed=set(['returning', 'where',
                              'strict']))
    if returning:
        query_l.append(make_returning_clause(returning))
    queries = [' '.join(query_l)+';']
    query = ''.join(queries)
    if query_data is None:
        query_data = {}
    return query, query_data
    # -------------------------------------------- ] ... delete ]


def select(table,  # ----------------------------- [ select ... [
           *args, **kwargs):
    """
    Return a simple "SELECT ..." SQL statement with placeholders
    and a values dict.

    Arguments:

      table -- name of the table (may contain the schema)
      fields -- '*' by default; other values must not be strings
               (but sequences of strings).
      query_data -- a dict to restrict the returned rows.

    For convenience, both fields and query_data can be specified named or unnamed.
    Keyword arguments are checked first; missing specifications are taken from the anonymous arguments.

    Keyword-only options:

      where -- a "WHERE ..." string (which may contain placeholders);
               currently necessary if some fields are both changed
               and used in the filtering `query_data`
      distinct -- if True, the DISTINCT keyword is inserted right after SELECT
                  (default: False)

    There is currently no JOIN, ORDER BY etc. support whatsoever;
    if you you need this, please use views.

    >>> tup1 = select('the.table', {'status': 'done', 'num': 42})
    >>> tup1[0]
    'SELECT * FROM the.table WHERE num = %(num)s AND status = %(status)s;'
    >>> tup1[1]
    {'status': 'done', 'num': 42}

    >>> select('another.table')[0]
    'SELECT * FROM another.table;'

    Table names are checked for invalid values:
    >>> select('another.table;truncate table users')[0]
    Traceback (most recent call last):
    ...
    ValueError: Invalid chars in 'another.table;truncate table users': (' ', ';')

    Field names are checked for invalid values:
    >>> select('another.table', {'evil_field;truncate table users': 42})[0]
    Traceback (most recent call last):
    ...
    ValueError: Invalid chars in 'evil_field;truncate table users': (' ', ';')

    Often we are interested in a particular subset of the fields:
    >>> select('another.table', {'status': 'done', 'num': 42}, fields=['id', 'changed'])[0]
    'SELECT id, changed FROM another.table WHERE num = %(num)s AND status = %(status)s;'

    The fields can as well be given as a single string (joined by whtiespace):
    >>> select('yet_another.table', 'id changed', {'status': 'done', 'num': 43})[0]
    'SELECT id, changed FROM yet_another.table WHERE num = %(num)s AND status = %(status)s;'

    If both fields and the query_data are given ancnymously, the fields are preferred,
    which is important when a None value is given:
    >>> select('yet_another.table', None, 'id changed')
    Traceback (most recent call last):
    ...
    TypeError: Duplicate fields spec. ('id changed'; we already have None)

    >>> select('yet_another.table', None, None, None)
    Traceback (most recent call last):
    ...
    TypeError: Surplus anonymous argument None

    >>> select('fancy.table', None, {})
    ('SELECT * FROM fancy.table;', {})

    >>> select('yet_another.table', None, None)
    ('SELECT * FROM yet_another.table;', {})

    >>> select('some_boring.table', 'id', distinct=True)
    ('SELECT DISTINCT id FROM some_boring.table;', {})

    For unknown keyword arguments, we get a TypeError:
    >>> select('fancy.table', unknown=42)
    Traceback (most recent call last):
    ...
    TypeError: Unknown option 'unknown' found!
    """
    fields, query_data = _fields_and_querydata(args, kwargs)

    pop = kwargs.pop
    where = pop('where', None)
    distinct = pop('distinct', False)
    check_kwargs(kwargs)

    query_l = ['SELECT',
               fields,
               replace_names('FROM %(table)s', table=table),
               ]
    if distinct:
        query_l.insert(1, 'DISTINCT')
    if where and not isinstance(where, six_string_types):
        raise ValueError('where must be a string; found %(where)r'
                         % locals())
    if where is None and query_data:
        where = make_where_mask(query_data, fields)
    if where:
        query_l.append(where)
    query = ' '.join(query_l) + ';'
    if query_data is None:
        query_data = {}
    return query, query_data
    # -------------------------------------------- ] ... select ]


def query(clause, *args, **kwargs):  # [ ---------- [ query ... [
    r"""
    Prepare an arbitrary SQL statement

    Arguments:

      clause -- a string, containing the SQL code to transform
                (may contain any number of statements).
                May contain %(name)s placeholders.
                NOTE: This code is not parsed or checked in any way;
                      don't use code from untrusted sources,
                      most notably: user input!
      names -- a dictionary of *names* which are
               replaced by this function
      query_data -- a dictionary which is used by the database adapter
                    to replace values.

    >>> txt = 'SELECT t1.user_id, t1.name, t2.address ' \
    ...         'FROM %(table1)s t1 ' \
    ...         'JOIN %(table2)s t2 ' \
    ...           'ON t1.user_id = t2.user_id ' \
    ...        "WHERE t1.name ILIKE %(filter)s;"
    >>> query(txt,
    ...       {'table1': 'users', 'table2': 'addresses'},
    ...       {'filter': '%beeblebrox%'})
    ('SELECT t1.user_id, t1.name, t2.address FROM users t1 JOIN addresses t2 ON t1.user_id = t2.user_id WHERE t1.name ILIKE %(filter)s;', {'filter': '%beeblebrox%'})

    The placeholders for the values are not directly replaced by the values
    (which is the task of the database adapter)
    but transformed to the requested convention
    (which is a NO-OP in the case of psychopg2).

    The difference is important: names and values are quoted differently
    in SQL; for values, single quotes are used ('), and double quotes (")
    for names.  Thus, if you don't treat the names specially, you'll get
    something which the database server will very likely reject:

    >>> query(txt,
    ...       query_data={  # BAD EXAMPLE!
    ...        'table1': 'users', 'table2': 'addresses',
    ...        'filter': '%beeblebrox%'})
    ('SELECT t1.user_id, t1.name, t2.address FROM %(table1)s t1 JOIN %(table2)s t2 ON t1.user_id = t2.user_id WHERE t1.name ILIKE %(filter)s;', {'table2': 'addresses', 'filter': '%beeblebrox%', 'table1': 'users'})

    The database adapter will probably replace the %(table1)s string by
    'users', but here a name is expected,
    and this must be double quoted ("users"); thus,
    a standards-complying SQL server will reject this.

    If you have your names replaced already, you can easily skip the names
    argument, of course:

    >>> txt = 'SELECT t1.user_id, t1.name, t2.address ' \
    ...         'FROM users t1 ' \
    ...         'JOIN addresses t2 ' \
    ...           'ON t1.user_id = t2.user_id ' \
    ...        "WHERE t1.name ILIKE %(filter)s;"
    >>> query(txt,
    ...       None,
    ...       {'filter': '%beeblebrox%'})
    ('SELECT t1.user_id, t1.name, t2.address FROM users t1 JOIN addresses t2 ON t1.user_id = t2.user_id WHERE t1.name ILIKE %(filter)s;', {'filter': '%beeblebrox%'})

    In some situations the aliases (t1, t2 in the above example) won't work,
    e.g. when using GROUP BY and HAVING;
    you can avoid them by using the names dictionary:

    >>> txt = 'SELECT %(table1)s.user_id, %(table1)s.name, %(table2)s.address' \
    ...        ' FROM %(table1)s ' \
    ...         'JOIN %(table2)s ' \
    ...           'ON %(table1)s.user_id = %(table2)s.user_id ' \
    ...        "WHERE %(table1)s.name ILIKE %(filter)s;"
    >>> query(txt,
    ...       {'table1': 'users', 'table2': 'addresses'},
    ...       {'filter': '%beeblebrox%'})[0]
    'SELECT users.user_id, users.name, addresses.address FROM users JOIN addresses ON users.user_id = addresses.user_id WHERE users.name ILIKE %(filter)s;'

    Some keyword arguments which are useful for other function are
    explicitly rejected:

    >>> query(txt,
    ...       {'table1': 'users', 'table2': 'addresses'},
    ...       {'filter': '%beeblebrox%'},
    ...       fields=['user_id', 'address'])
    Traceback (most recent call last):
      ...
    TypeError: 'fields' argument not supported; we don't build SQL statements here.

    This is done as a hint for this function being a quite simple transformer;
    building arbitrary statements is beyond it's scope.
    """
    for misplaced in ['distinct', 'fields', 'where']:
        if misplaced in kwargs:
            raise TypeError('%(misplaced)r argument not supported; '
                            'we don\'t build SQL statements here.'
                            % locals())
    names, query_data = _some_dicts(args, kwargs, 'names query_data')
    if query_data is None:
        query_data = {}
    if names is None:
        query = replace_names(clause, **kwargs)
    else:
        check_kwargs(kwargs)
        query = replace_names(clause, **names)
    return query, query_data
    # --------------------------------------------- ] ... query ]
# ---------------------------------- ] ... SQL-Statements generieren ]


if __name__ == '__main__':
    # Standard library:
    import doctest
    doctest.testmod()

# vim: ts=8 sts=4 sw=4 si et
