# pylint: disable=missing-function-docstring
# pylint: disable=missing-class-docstring
# pylint: disable=line-too-long
# pylint: disable=unused-wildcard-import
# pylint: disable=wildcard-import
# pylint: disable=unused-import
'''
    A module of utility methods used for parsing SQL code.

    ----------

    Meta
    ----------
    `author`: Colemen Atwood
    `created`: 06-03-2022 10:04:06
    `memberOf`: parse_sql
    `version`: 1.0
    `method_name`: parse_sql
'''


import re as _re
import json as _json
import shlex as _shlex
import sqlparse as _sqlparse
import yaml
from typing import Union as _Union

from pyparsing import And, Suppress,Dict, Word, Literal, Group, Optional, ZeroOrMore, OneOrMore, Regex, restOfLine, alphanums,nums, printables, string, CaselessKeyword,nestedExpr,ParseException,quotedString,removeQuotes,originalTextFor,delimitedList,QuotedString

import colemen_utilities.dict_utils as _obj
import colemen_utilities.list_utils as _lu
import colemen_utilities.string_utils as _csu
import colemen_utilities.file_utils as _f
import colemen_utilities.sql_utils as _sql
import colemen_utilities.console_utils as _console
# import colemen_utilities.sql_utils.TableStatement as _table_statement
# import colemen_config as _config
_log = _console.log


_sql_data_types = [
    "mediumtext",
    "mediumblob",
    "varbinary",
    "timestamp",
    "mediumint",
    "tinytext",
    "tinyblob",
    "smallint",
    "longtext",
    "longblob",
    "datetime",
    "varchar",
    "tinyint",
    "integer",
    "decimal",
    "boolean",
    "double",
    "double",
    "binary",
    "bigint",
    "float",
    "float",
    "year",
    "time",
    "text",
    "enum",
    "date",
    "char",
    "bool",
    "blob",
    "set",
    "int",
    "dec",
    "bit",
]

_sql_types_with_sizes = [
    "varbinary",
    "mediumint",
    "timestamp",
    "smallint",
    "datetime",
    "varchar",
    "decimal",
    "integer",
    "tinyint",
    "binary",
    "double",
    "double",
    "bigint",
    "float",
    "float",
    "text",
    "char",
    "time",
    "blob"
    "dec",
    "int",
    "bit",
]


def is_valid_mysql_data_type(value:str):
    data = mysql_parse_type_data(value)
    if data is False or data['type'] is None:
        return False
    value = data['type']

    if value in _sql_data_types:
        return data

    value_lower = value.lower()
    if value_lower in _sql_data_types:
        data['type'] = value_lower
        return data

    varis = _csu.variations(value)
    for var in varis:
        if var in _sql_data_types:
            data["type"] = var
            return data
    return False

def mysql_parse_type_data(value:str):
    if isinstance(value,(str)) is False:
        return False
    value = value.lower()
    data = {
        "type":None,
        "size":None,
    }

    pat = _re.compile(r"([A-Za-z]*)(?:\(([0-9]*)\))?")
    match = _re.findall(pat,value)
    if match is not None:
        if len(match) > 0:
            type_name = match[0][0]
            type_size = match[0][1]
            # print(f"match[0]:{match[0]}")
            # print(f"    type_name:{type_name}")
            data['type'] = type_name
            if len(type_size) > 0:
                data['size'] = int(type_size)

    return data

def gen_mysql_type(name:str,size:int=None):
    name = name.lower()
    if size is not None:
        if name in _sql_types_with_sizes:
            return f"{name}({size})"
    if size is None:
        if name in _sql_data_types:
            return f"{name}"


def sql_type_to_python(value):
    '''
        Attempts to convert an sql type to its python equivalent.

        Keep in mind this is not exact.. at fucking all..
        This is attempting to predict how the value will need to be stored in the SQL file.

        ----------

        Arguments
        -------------------------
        `value` {str}
            The SQL type to convert.

        Return {list}
        ----------------------
        A list of similar types

        Meta
        ----------
        `author`: Colemen Atwood
        `created`: 06-02-2022 07:20:35
        `memberOf`: parse_sql
        `version`: 1.0
        `method_name`: sql_type_to_python
        * @xxx [06-02-2022 07:22:30]: documentation for sql_type_to_python
    '''

    if value in _sql_data_types:
        return _sql_data_types[value]
    return None






def _parse_statement_comment(value):
    '''
        A utility method used to parse the SQL COMMENT 'beepBoopBleepBlorp' portion of a string.

        This is intended for general use on table, column and index statements.

        ----------

        Arguments
        -------------------------
        `value` {str}
            The SQL statement to parse the comment from.

        Return {tuple}
        ----------------------
        A tuple containing the value with the comment removed and the comment itself.

        if no comment is found the second indice of the tuple is None: (value,None)

        `example`:\n
        SQL: \`modified_timestamp\`               int NULL DEFAULT NULL COMMENT 'description: The unix timestamp of when this was last modified, null otherwise.',\n
        result = ("\`modified_timestamp\`               int NULL DEFAULT NULL","description: The unix timestamp of when this was last modified, null otherwise.")


        SQL: ") COMMENT='description: Defines a registration association entity.';"\n
        result = (")","description: Defines a registration association entity.")


        Meta
        ----------
        `author`: Colemen Atwood
        `created`: 07-21-2022 08:59:41
        `memberOf`: sql_parse
        `version`: 1.0
        `method_name`: _parse_statement_comment
        * @xxx [07-21-2022 09:03:45]: documentation for _parse_statement_comment
    '''

    # ENGINE=InnoDB AUTO_INCREMENT=754 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
    # _f.write("tmp1.tmp.sql",value)

    # _f.write("tmp.tmp.sql",value)

    # @Mstep [] capture the comment if it is found in the line.
    comment_value = None
    match = _re.findall(r"(comment(?:=|\s)\'(.*)$)",value,_re.IGNORECASE)
    # match = _re.findall(r"(?:(comment(?:=|\s)\'(.*)$)|(\s+comment\s+\'(.*)$))",value,_re.IGNORECASE)
    # match = _re.findall(r"(comment(?:=|\s)\'(.*)$)",value,_re.IGNORECASE)
    if match is not None and len(match) > 0:
        match = list(match[0])
        comment_value = str(match[1])
        comment_value = _csu.strip(comment_value,["'",'"',","," ",";"])
        # print(f"table comment found: {comment_value}")
        value = value.replace(match[0],"")

    return (value,comment_value)




def parse_column_data(value=None):

    # value = f"`theme` varchar(45) COLLATE utf8_unicode_ci DEFAULT 'dark' COMMENT 'description: The name of theme the user wants the frontend to use.\r\noptions:\r\n- no_sus_get\r\n- valueOptions: [dark,light]\r\n- updateForm: UpdateUserSettings\r\n- input_type: toggle',"
    # value = f"`is_public` tinyint(4) DEFAULT '1' COMMENT 'description: 1 if the profile is publicly visible/searchable, 0 otherwise\r\noptions:\r\n- boolOpt\r\n- updateForm: UpdateUserSettings\r\n- no_sus_get\r\n- update_sus: true\r\n- input_type: toggle',"
    # value = f"`vid_autoplay_mobile` tinyint(4) DEFAULT '1' COMMENT 'description: 1 if the user wants videos to autoplay on mobile devices.\r\noptions:\r\n- boolOpt\r\n- updateForm: UpdateUserSettings\r\n- update_sus: true\r\n- input_type: toggle',"


    value = _re.sub(r"COLLATE\s?(?:utf8_unicode_ci)"," ",value)
    value = value.replace("ENGINE=InnoDB","")
    value = value.replace("DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci","")
    value = value.replace("DEFAULT NULL"," NULL DEFAULT NULL")
    value = _re.sub(r"\s+"," ",value)

    # print(value)

    # @Mstep [] capture the comment if it is found in the line.
    value, comment_value = _parse_statement_comment(value)
    # match = _re.findall(r"(\s+comment\s+\'(.*)$)",value,_re.IGNORECASE)
    # # match = _re.findall(r"(\s+COMMENT\s+([\'\"][^\']*[\'\"]|[0-9\.]))",value,_re.IGNORECASE)
    # if match is not None and len(match) > 0:
    #     match = list(match[0])
    #     comment_value = str(match[1])
    #     comment_value = _csu.strip(comment_value,["'",'"',","," "])
    #     value = value.replace(match[0],"")


    # @Mstep [] capture the default value if it is found in the line.
    default_value = None
    match = _re.findall(r"(\s+default\s+(.*)$)",value,_re.IGNORECASE)
    if match is not None and len(match) > 0:
        match = list(match[0])
        default_value = _csu.strip(match[1]," ")

        if _re.match(r"^[0-9\+-\.]*$",default_value) is not None:
            default_value = int(default_value)
        value = value.replace(match[0],"")
        default_value = _csu.strip(default_value,["'"])

    # print(f"value: {value}")
    # print(f"default_value: {default_value}")







    # value = "`timestamp`       int NULL DEFAULT NULL COMMENT 'Timestamp of when the task was created.' ,"
    _tick,_double_quote,_single_quote,_comma = [Suppress(x) for x in list('`"\',')]
    _quote = (_tick|_double_quote|_single_quote)
    _open_paren,_close_paren = [Suppress(x) for x in list('()')]
    # paren_word = _open_paren + Optional(_quote) + Word(alphanums + "_") + Optional(_quote) + _close_paren
    quoted_word = _quote + Word(alphanums + "_") + _quote


    column_name = quoted_word.setResultsName('name')

    ctypes = CaselessKeyword("mediumtext") | CaselessKeyword("mediumblob") | CaselessKeyword("varbinary") | CaselessKeyword("timestamp") | CaselessKeyword("mediumint") | CaselessKeyword("tinytext") | CaselessKeyword("tinyblob") | CaselessKeyword("smallint") | CaselessKeyword("longtext") | CaselessKeyword("longblob") | CaselessKeyword("datetime") | CaselessKeyword("varchar") | CaselessKeyword("tinyint") | CaselessKeyword("integer") | CaselessKeyword("decimal") | CaselessKeyword("boolean") | CaselessKeyword("double") | CaselessKeyword("double") | CaselessKeyword("binary") | CaselessKeyword("bigint") | CaselessKeyword("float") | CaselessKeyword("float") | CaselessKeyword("year") | CaselessKeyword("time") | CaselessKeyword("text") | CaselessKeyword("enum") | CaselessKeyword("date") | CaselessKeyword("char") | CaselessKeyword("bool") | CaselessKeyword("blob") | CaselessKeyword("set") | CaselessKeyword("int") | CaselessKeyword("dec") | CaselessKeyword("bit")
    ctype_len = Optional(_open_paren + Word(nums) + _close_paren)
    column_type = ctypes.setResultsName('type') + ctype_len.setResultsName('size')

    null_vals = CaselessKeyword("NULL") | CaselessKeyword("NOT NULL")
    null_type = Optional(null_vals.setResultsName('null_type'))


    grammar = column_name + column_type + null_type
    res = ZeroOrMore(Group(grammar).setResultsName('data')).parseString(value)

    output = None

    if len(res) > 0:
        data = res.as_dict()['data']
        # print(data)
        defaults = {
            "name":None,
            "type":None,
            "allow_nulls":None,
            "default":default_value,
            "comment":comment_value,
        }
        new_data = {}
        for k,v in data.items():
            if k == "size":
                new_data[k] = int(v[0])
                continue
            if k == "null_type":
                if v.upper() == 'NOT NULL':
                    new_data['allow_nulls'] = False
                    continue
                if v.upper() == 'NULL':
                    new_data['allow_nulls'] = True
                    continue
            if isinstance(v,(list)):
                v = _csu.strip(v[0],["'",'"'])
                new_data[k] = v
                continue
            new_data[k] = v

        # print(new_data)
        new_data = _obj.set_defaults(defaults,new_data)
        output = new_data
    # print(output)
    return output

def parse_schema_statement(value=None):

    # value = "CREATE SCHEMA IF NOT EXISTS `idealech_Equari_Content_Database`;"


    _tick,_double_quote,_single_quote,_period = [Suppress(x) for x in list('`"\'.')]
    _quote = (_tick|_double_quote|_single_quote)
    _open_paren,_close_paren = [Suppress(x) for x in list('()')]
    # paren_word = _open_paren + Optional(_quote) + Word(alphanums + "_") + Optional(_quote) + _close_paren
    quoted_word = _quote + Word(alphanums + "_") + _quote

    keys = CaselessKeyword("schema")
    create_statement = CaselessKeyword("create").setResultsName('action') + keys
    drop_statement = CaselessKeyword("drop").setResultsName('action') + keys
    action = drop_statement | create_statement


    exists = Optional(CaselessKeyword("if exists") | CaselessKeyword("if not exists")).setResultsName('test')
    schema_name = quoted_word.setResultsName('schema_name')


    grammar = action + exists + schema_name
    res = ZeroOrMore(Group(grammar).setResultsName('data')).parseString(value)

    output = None

    if len(res) > 0:
        data = res.as_dict()['data']
        # print(data)
        defaults = {
            "action":None,
            "test":None,
            "schema_name":None,
            "raw_statement":value,
        }
        new_data = {}
        for k,v in data.items():
            if isinstance(v,(list)):
                new_data[k] = v[0]
                continue
            new_data[k] = v
        new_data = _obj.set_defaults(defaults,new_data)
        # print(new_data)
        output = new_data
    return output

def parse_table_statement(value:_Union[None,str]=None)->_Union[dict,None]:
    '''
        Parse a table SQL statement.

        ----------

        Arguments
        -------------------------
        `value` {str,None}
            The SQL string to parse.

        Return {dict,None}
        ----------------------
        The statement data dictionary if it can be parsed, None otherwise.

        {
            `action`: "create",
            `test`: "if not exists",
            `schema_name`: "idealech_Equari_Management_Database",
            `table_name`: "faq_categories",
            `raw_statement`: "Original and Unmodified Create Statement Goes here.",
            `columns`: [
                {
                    "name": "faq_category_id",
                    "type": "int",
                    "allow_nulls": false,
                    "default": null,
                    "comment": null,
                    "is_primary_key": false,
                    "primary_key": true
                },
                ...
            ],
            `primary_keys`: [
                "faq_category_id"
            ],
            `keys`: [
                {
                    "key_name": "FK_FaqCategories_FaqCategoryID",
                    "foreign_col_name": [
                        "parent_faq_category_id"
                    ]
                }
            ],
            `constraints`: [
                {
                    "constraint_name": "FK_FaqCategory_Parent_FaqCategory",
                    "foreign_key": "FK_FaqCategories_FaqCategoryID",
                    "local_col_name": "parent_faq_category_id",
                    "schema_name": "idealech_Equari_Management_Database",
                    "table_name": "faq_categories",
                    "foreign_col_name": "faq_category_id",
                    "on_delete": "CASCADE",
                    "on_update": "CASCADE"
                }
            ],
            `name`: "faq_categories"
        }



        Meta
        ----------
        `author`: Colemen Atwood
        `created`: 06-05-2022 20:30:22
        `memberOf`: sql_parse
        `version`: 1.0
        `method_name`: parse_table_statement
        * @xxx [06-05-2022 20:34:11]: documentation for parse_table_statement
    '''
    # _log("sql_parse.parse_table_statement")

    # @Mstep [] capture the comment if it is found in the line.
    value,comment_value = _parse_statement_comment(value)
    # match = _re.findall(r"(comment=\'(.*)$)",value,_re.IGNORECASE)
    # if match is not None and len(match) > 0:
    #     match = list(match[0])
    #     comment_value = str(match[1])
    #     comment_value = _csu.strip(comment_value,["'",'"',","," ",";"])
    #     # print(f"table comment found: {comment_value}")
    #     value = value.replace(match[0],"")


    _tick,_double_quote,_single_quote,_period = [Suppress(x) for x in list('`"\'.')]
    _quote = (_tick|_double_quote|_single_quote)
    # _open_paren,_close_paren = [Suppress(x) for x in list('()')]
    # paren_word = _open_paren + Optional(_quote) + Word(alphanums + "_") + Optional(_quote) + _close_paren
    quoted_word = _quote + Word(alphanums + "_") + _quote


    keys = CaselessKeyword("table")
    create_statement = CaselessKeyword("create").setResultsName('action') + keys
    drop_statement = CaselessKeyword("drop").setResultsName('action') + keys
    action = drop_statement | create_statement


    exists = Optional(CaselessKeyword("if exists") | CaselessKeyword("if not exists")).setResultsName('test')
    table_name = Optional(quoted_word.setResultsName('schema_name') + _period) + quoted_word.setResultsName('table_name')


    grammar = action + exists + table_name
    res = ZeroOrMore(Group(grammar).setResultsName('data')).parseString(value)

    output = None

    if len(res) > 0:
        data = res.as_dict()['data']
        defaults = {
            "raw_statement":value,
            "action":None,
            "test":None,
            "schema_name":None,
            "table_name":None,
            "comment":comment_value,
        }
        new_data = {}
        for k,v in data.items():
            if isinstance(v,(list)):
                new_data[k] = v[0]
                continue
            new_data[k] = v
        new_data = _obj.set_defaults(defaults,new_data)

        if new_data['action'] == "create":
            cols = capture_create_table_columns(value)
            # new_data['column_data'] = _obj.strip_list_nulls([parse_column_data(x) for x in cols])
            d = _parse_table_column_lines(cols)
            new_data = {**new_data,**d}
            # new_data["columns"] = d['columns']
            # new_data["primary_keys"] = d['primary_keys']
            # new_data["unique_keys"] = d['unique_keys']
            # new_data["keys"] = d['keys']
            # new_data["constraints"] = d['constraints']
            new_data["comment"] = comment_value
        # print(new_data)
        output = new_data

    return output

# def _parse_table_column_lines(lines):
#     data = {
#         "columns":[],
#         "primary_keys":[],
#         "indexes":[],
#         "unique_keys":[],
#         "keys":[],
#         "constraints":[],
#     }
#     # keys = []

#     for line in lines:
#         # print(f"line: {line}")
#         if is_line_key(line):
#             # print(f"key found: {line} {parse_key(line)}")
#             data['keys'] = _lu.append(data['keys'],parse_key(line))

#             ft = parse_fulltext_key(line)
#             if ft is not None:
#                 data['indexes'].append(ft)

#             uk = parse_unique_key(line)
#             if uk is not None:
#                 # print(f"Unique Key Found: {uk}")
#                 data['indexes'].append(uk)
#                 data['unique_keys'].append(uk)
#                 # data['unique_keys'] = _lu.append(data['unique_keys'],uk['unique_key'])

#             pk = parse_primary_key(line)
#             if pk is not None:
#                 data['indexes'].append(pk)
#                 data['primary_keys'] = _lu.append(data['primary_keys'],pk['primary_key'])
#             # data['primary_keys'] = _lu.append(data['primary_keys'],parse_primary_key(line))

#         if is_line_constraint(line):
#             data['constraints'] = _lu.append(data['constraints'],parse_constraint(line))


#         data['columns'] = _lu.append(data['columns'],parse_column_data(line))

#         # parse_column_data(line)
#     # data['keys'] = keys
#     for c in data['columns']:
#         c['is_primary_key'] = False
#         if c['name'] in data['primary_keys']:
#             c['is_primary_key'] = True



#     return data

def capture_create_table_columns(sql):
    '''
        Used by parse_table_statement to capture the column area of the statement.

        ----------

        Arguments
        -------------------------
        `sql` {str}
            The create table statement to parse.

        Return {list}
        ----------------------
        A list of column declarations upon success.
        The list is empty if nothing is found.

        Meta
        ----------
        `author`: Colemen Atwood
        `created`: 06-01-2022 08:48:11
        `memberOf`: parse_sql
        `version`: 1.0
        `method_name`: capture_create_table_columns
        @xxx [06-01-2022 08:50:01]: documentation for capture_create_table_columns
    '''

    output = []
    scanner = originalTextFor(nestedExpr('(',')'))
    for match in scanner.searchString(sql):
        val = _csu.strip(match[0],["(",")"])
        val = _sql.escape_quoted_chars(val,True,['__%0A__'])
        # print(f"val: {val}")
        output = val.split("\n")
        # newlist = [_log(x,"blue") for x in output]
        # output = [_sql.escape_quoted_chars(x,True) for x in output]
        # newlist = [_log(x,"yellow") for x in output]
    return output


def _parse_list(value:str)->str:
    '''
        A utility function used for parsing an SQL string list into a more formatted version.

        ----------

        Arguments
        -------------------------
        `arg_name` {type}
                arg_description

        Keyword Arguments
        -------------------------
        `arg_name` {type}
                arg_description

        Return {type}
        ----------------------
        return_description

        Meta
        ----------
        `author`: Colemen Atwood
        `created`: 06-05-2022 20:27:02
        `memberOf`: sql_parse
        `version`: 1.0
        `method_name`: _parse_list
        * @xxx [06-05-2022 20:28:56]: documentation for _parse_list
    '''


    value = value.asList()
    value = _sql.strip_sql_quotes(value[0])
    vals = value.split(",")
    vals = [_csu.strip(x,[" "]) for x in vals]
    return ','.join(vals)

# def parse_key(value=None):
#     if 'key' not in value.lower():
#         return None
#     # value ="KEY `FK_ActivityLogs_ActivityTypeID` (`activity_type_id`, `activity_type_hash_id`),"
#     _tick,_double_quote,_single_quote,_comma = [Suppress(x) for x in list('`"\',')]
#     _quote = (_tick|_double_quote|_single_quote)
#     _open_paren,_close_paren = [Suppress(x) for x in list('()')]
#     # paren_word = _open_paren + Optional(_quote) + Word(alphanums + "_") + Optional(_quote) + _close_paren
#     quoted_word = _quote + Word(alphanums + "_") + _quote

#     paren_word_list = _open_paren + Word(alphanums + "_,'\"` ") + _close_paren

#     # KEY `FK_ActivityLogs_RequestLogID`
#     key_name = Optional(CaselessKeyword('unique')) + Suppress(CaselessKeyword('KEY')) + quoted_word.setResultsName('key_name')

#     foreign_col_name = paren_word_list.setResultsName('foreign_col_name').setParseAction(_parse_list)

#     grammar = key_name + foreign_col_name
#     res = ZeroOrMore(Group(grammar).setResultsName('data')).parseString(value)

#     output = None

#     if len(res) > 0:
#         data = res.as_dict()['data']
#         new_data = {}
#         for k,v in data.items():
#             if k == "foreign_col_name":
#                 new_data[k] = v.split(",")
#                 continue
#             if isinstance(v,(list)):
#                 new_data[k] = v[0]
#                 continue
#             new_data[k] = v

#         # print(new_data)
#         output = new_data
#     return output

# def parse_primary_key(value=None):
#     if 'primary key' not in value.lower():
#         return None
#     # value = "PRIMARY KEY (`activity_log_id`)"
#     # value = "PRIMARY KEY (`activity_type_id`, `activity_type_hash_id`),"
#     _tick,_double_quote,_single_quote,_period = [Suppress(x) for x in list('`"\'.')]
#     _quote = (_tick|_double_quote|_single_quote)
#     _open_paren,_close_paren = [Suppress(x) for x in list('()')]
#     # paren_word = _open_paren + Optional(_quote) + Word(alphanums + "_") + Optional(_quote) + _close_paren
#     # quoted_word = _quote + Word(alphanums + "_") + _quote
#     paren_word_list = _open_paren + Word(alphanums + "_,'\"` ") + _close_paren
#     primary_key = paren_word_list.setResultsName('primary_key').setParseAction(_parse_list)
#     grammar = Suppress(CaselessKeyword('primary key')) + primary_key
#     res = ZeroOrMore(Group(grammar).setResultsName('data')).parseString(value)

#     output = None

#     if len(res) > 0:
#         data = res.as_dict()['data']
#         new_data = {
#             "index_type":"primary_key"
#         }
#         for k,v in data.items():
#             if k == "primary_key":
#                 new_data[k] = v.split(",")
#                 continue
#             if isinstance(v,(list)):
#                 new_data[k] = v[0]
#                 continue
#             new_data[k] = v

#         # print(new_data)
#         output = new_data
#     return output


# def parse_unique_key(value=None):

#     if 'unique key' not in value.lower():
#         return None
# # UNIQUE KEY `Unique_barnBlocks_barnID_userID_3006` (`user_id`, `barn_id`) COMMENT 'Unique constraint to ensure a barn cannot block the same user multiple times.',

#     _tick,_double_quote,_single_quote,_period = [Suppress(x) for x in list('`"\'.')]
#     _quote = (_tick|_double_quote|_single_quote)
#     _open_paren,_close_paren = [Suppress(x) for x in list('()')]
#     # paren_word = _open_paren + Optional(_quote) + Word(alphanums + "_") + Optional(_quote) + _close_paren
#     quoted_word = _quote + Word(alphanums + "_") + _quote


#     paren_word_list = _open_paren + Word(alphanums + "_,'\"` ") + _close_paren
#     unique_key = paren_word_list.setResultsName('columns').setParseAction(_parse_list)
#     key_name = quoted_word.setResultsName('constraint_name')

#     comment = None
#     match = _re.findall(r"comment\s*'([^']*)",value,_re.IGNORECASE)
#     if isinstance(match,(list)) and len(match) > 0:
#         comment = match[0]

#     grammar = Suppress(CaselessKeyword('unique key')) + key_name + unique_key
#     res = ZeroOrMore(Group(grammar).setResultsName('data')).parseString(value)

#     output = None

#     if len(res) > 0:
#         data = res.as_dict()['data']

#         new_data = {
#             "index_type":"unique",
#             "comment":comment,
#         }
#         for k,v in data.items():
#             if k == "columns":
#                 new_data[k] = v.split(",")
#                 continue
#             if isinstance(v,(list)):
#                 new_data[k] = v[0]
#                 continue
#             new_data[k] = v
#         # print(f"Unique Key Data: {new_data}")
#         # print(new_data)
#         output = new_data
#     return output

# def parse_fulltext_key(value=None):
#     # value = "FULLTEXT KEY `FullText_regAss_name_3079` (`name`) COMMENT 'A fullText Index on the registration association name for searching.'"

#     value = _re.sub(r'full[^a-z]*text[^a-z]*key','fulltext key',value,_re.IGNORECASE)
#     if 'fulltext key' not in value.lower():
#         return None

#     value,comment_value = _parse_statement_comment(value)
#     # print(f"after comment: {value}")

#     # FULLTEXT KEY `FullText_regAss_name_3079` (`name`) COMMENT 'A fullText Index on the registration association name for searching.'

#     _tick,_double_quote,_single_quote,_period = [Suppress(x) for x in list('`"\'.')]
#     _quote = (_tick|_double_quote|_single_quote)
#     _open_paren,_close_paren = [Suppress(x) for x in list('()')]
#     # paren_word = _open_paren + Optional(_quote) + Word(alphanums + "_") + Optional(_quote) + _close_paren

#     # matches : `FullText_regAss_name_3079`
#     quoted_word = _quote + Word(alphanums + "_") + _quote


#     paren_word_list = _open_paren + Word(alphanums + "_,'\"` ") + _close_paren
#     column_list = paren_word_list.setResultsName('columns').setParseAction(_parse_list)
#     key_name = quoted_word.setResultsName('constraint_name')

#     # comment = None
#     # match = _re.findall(r"comment\s*'([^']*)",value,_re.IGNORECASE)
#     # if isinstance(match,(list)) and len(match) > 0:
#     #     comment = match[0]

#     grammar = Suppress(CaselessKeyword('fulltext key')) + key_name + column_list
#     res = ZeroOrMore(Group(grammar).setResultsName('data')).parseString(value)

#     output = None

#     if len(res) > 0:
#         data = res.as_dict()['data']

#         new_data = {
#             "index_type":"fulltext",
#             "comment":comment_value,
#         }

#         for k,v in data.items():
#             if k == "columns":
#                 new_data[k] = v.split(",")
#                 continue
#             if isinstance(v,(list)):
#                 new_data[k] = v[0]
#                 continue
#             new_data[k] = v
#         # print(f"fulltext Key Data: {new_data}")
#         # print(new_data)
#         output = new_data
#     return output

# def parse_constraint(value:str):
#     '''
#         Parse constraint data from the string.

#         ----------

#         Arguments
#         -------------------------
#         `value` {str}
#             The string to parse.

#         Return {dict,None}
#         ----------------------
#         The constraint data dictionary if successful, None otherwise.

#         Meta
#         ----------
#         `author`: Colemen Atwood
#         `created`: 06-05-2022 20:20:57
#         `memberOf`: sql_parse
#         `version`: 1.0
#         `method_name`: parse_constraint
#         * @xxx [06-05-2022 20:26:05]: documentation for parse_constraint
#     '''


#     _tick,_double_quote,_single_quote,_period = [Suppress(x) for x in list('`"\'.')]
#     _quote = (_tick|_double_quote|_single_quote)
#     _open_paren,_close_paren = [Suppress(x) for x in list('()')]
#     paren_word = _open_paren + Optional(_quote) + Word(alphanums + "_") + Optional(_quote) + _close_paren
#     quoted_word = _quote + Word(alphanums + "_") + _quote


#     constraint_conditions = (CaselessKeyword('RESTRICT') | CaselessKeyword('CASCADE') | CaselessKeyword('SET NULL') | CaselessKeyword('NO ACTION') | CaselessKeyword('SET DEFAULT'))


#     constraint_name = Suppress(CaselessKeyword('CONSTRAINT')) + quoted_word.setResultsName('constraint_name')


#     foreign_key_name = Suppress(CaselessKeyword('FOREIGN KEY')) + Optional(quoted_word.setResultsName('foreign_key'))
#     local_col_name = paren_word.setResultsName('local_col_name')
#     foreign_table = Suppress(CaselessKeyword('REFERENCES')) + Optional(quoted_word.setResultsName('schema_name') + _period) + quoted_word.setResultsName('table_name')
#     foreign_col_name = paren_word.setResultsName('foreign_col_name')

#     on_delete = Suppress(CaselessKeyword('ON DELETE')) + constraint_conditions.setResultsName('on_delete')
#     on_update = Suppress(CaselessKeyword('ON UPDATE')) + constraint_conditions.setResultsName('on_update')
#     constraints = ZeroOrMore(on_delete | on_update)

#     grammar = constraint_name + foreign_key_name + local_col_name + foreign_table + foreign_col_name + constraints
#     res = ZeroOrMore(Group(grammar).setResultsName('data')).parseString(value)

#     output = None

#     comment = None
#     match = _re.findall(r"comment\s*'([^']*)",value,_re.IGNORECASE)
#     if isinstance(match,(list)) and len(match) > 0:
#         comment = match[0]

#     if len(res) > 0:
#         data = res.as_dict()['data']
#         new_data = {
#             "comment":comment
#         }
#         for k,v in data.items():
#             if isinstance(v,(list)):
#                 new_data[k] = v[0]
#                 continue
#             new_data[k] = v

#         # print(new_data)
#         output = new_data
#     return output



def is_line_key(line:str)->bool:
    '''
        Test if the line contains a key constraint or statement.

        ----------

        Arguments
        -------------------------
        `line` {str}
            The string to parse.

        Return {bool}
        ----------------------
        True upon success, false otherwise.

        Meta
        ----------
        `author`: Colemen Atwood
        `created`: 06-05-2022 20:16:42
        `memberOf`: sql_parse
        `version`: 1.0
        `method_name`: is_line_key
        * @xxx [06-05-2022 20:17:55]: documentation for is_line_key
    '''

    match = _re.match(r'^(\w+\s*)?key',line,_re.IGNORECASE)
    if match is not None:
        return True
    return False

def is_line_comment(line:str)->bool:
    '''
        Determine if the line is commented.

        ----------

        Arguments
        -------------------------
        `line` {str}
            The string to parse.

        Return {bool}
        ----------------------
        True if the line is commented, False otherwise.

        Meta
        ----------
        `author`: Colemen Atwood
        `created`: 06-05-2022 20:13:33
        `memberOf`: sql_parse
        `version`: 1.0
        `method_name`: is_line_comment
        * @xxx [06-05-2022 20:14:45]: documentation for is_line_comment
    '''

    match = _re.match(r'^--',line,_re.IGNORECASE)
    if match is not None:
        return True
    return False

def is_line_constraint(line:str)->bool:
    '''
        Determine if the line provided contains a constraint.
        This is used to quickly test the strings before actually parsing, just to save time.

        ----------

        Arguments
        -------------------------
        `line` {str}
            The string to parse.

        Return {bool}
        ----------------------
        True if it contains a constraint, False otherwise.

        Meta
        ----------
        `author`: Colemen Atwood
        `created`: 06-05-2022 20:11:27
        `memberOf`: sql_parse
        `version`: 1.0
        `method_name`: is_line_constraint
        * @xxx [06-05-2022 20:13:17]: documentation for is_line_constraint
    '''


    match = _re.match(r'^constraint',line,_re.IGNORECASE)
    if match is not None:
        return True
    return False


def _parse_comment_yaml(contents):
    if len(contents) == 0:
        return None
    if isinstance(contents,(str)) is False:
        contents = ""

    # contents = contents.replace("\n","   ")
    contents = contents.replace("desc:","description:")
    contents = contents.replace("opts:","options:")
    contents = contents.replace("o:","options:")

    # contents = contents.replace("options:","options:\n")



    if len(contents) > 0:
        if contents.startswith("description:") is False:
            contents = f"description: {contents}"

        if "description:" not in contents:
            contents = f"description: {contents}"
    else:
        return None
    # @Mstep [] force a space between a dash and alphanum characters.
    contents = _re.sub(r"\n-([a-zA-Z0-9])",r"\n- \1",contents)
    # @Mstep [] force a space between a key's colon and the value following it.
    contents = _re.sub(r"(\n-\s*[a-zA-Z0-9_ ]*:\s*)",r"\1 ",contents)

    try:
        # result = _parse_comment_yaml(comment)
        data = yaml.safe_load(contents)
    except yaml.scanner.ScannerError:
        return None
    # c.con.log(f"contents: {contents}","red")
    output = {}
    if "description" in data:
        output['description'] = data['description']

    if "options" in data:
        if data['options'] is not None:
            for opt in data['options']:
                if isinstance(opt,(str)):
                    output[_csu.to_snake_case(opt)] = True
                if isinstance(opt,(dict)):
                    for k,value in opt.items():
                        output[_csu.to_snake_case(k)] = value
    # print(output)
    return output





def get_statements(sql:str):
    '''
        Parses the SQL statements from the string.

        ----------

        Arguments
        -------------------------
        `sql` {string}
            The sql string to parse.

        Return {list}
        ----------------------
        A list of SQL statements.

        Meta
        ----------
        `author`: Colemen Atwood
        `created`: 03-21-2022 12:40:52
        `version`: 1.0
        `method_name`: get_statements
        @xxx [03-21-2022 12:40:58]: documentation for get_statements
    '''


    raw = sql
    if isinstance(sql,(str)):
        if _f.exists(sql):
            raw = _f.readr(sql)
    # @Mstep [] remove single line comments from the sql.
    raw = _sql.strip_comments(raw)
    # @Mstep [] escape special characters that are within quotes.
    raw = _sql.escape_quoted_chars(raw)
    # raw = format_sql_string(raw)
    statements = [x for x in _sqlparse.parse(raw)]
    return statements

def determine_statement_purpose(statement:str):
    '''
        Determine the purpose of the SQL statement.

        ----------

        Arguments
        -------------------------
        `statement` {str}
            The SQL statement to parse.

        Return {str|None}
        ----------------------
        The statement's purpose or None

        Meta
        ----------
        `author`: Colemen Atwood
        `created`: 06-05-2022 20:08:22
        `memberOf`: sql_parse
        `version`: 1.0
        `method_name`: determine_statement_purpose
        * @xxx [06-05-2022 20:10:56]: documentation for determine_statement_purpose
    '''

    # _log("sql_parse.determine_statement_purpose")
    statement_types = ["CREATE DATABASE","ALTER DATABASE","CREATE SCHEMA","CREATE INDEX","CREATE TABLE","ALTER TABLE","INSERT INTO","DROP INDEX","DROP TABLE","DELETE","UPDATE","SELECT",]
    for s in statement_types:
        if s in statement.upper():
            return s

    return None

def parse(sql):
    # print("        sql_parse.parse")
    if _f.exists(sql):
        # print(f"    Reading SQL file: {sql}")
        sql = _f.readr(sql)

    if len(sql)== 0:
        return None

    sql = _csu.strip_empty_lines(sql)
    raw_statements = get_statements(sql)
    # print(f"raw_statements: {len(raw_statements)}")
    data = {
        "schemas":[],
        "tables":[],
        "statements":[],
    }

    for s in raw_statements:
        s = s.value
        # print(f"s: {s}")
        state = {
            "raw":s,
            "purpose":determine_statement_purpose(s),
            "data":None,
        }

        if state['purpose'] is None:
            _log(f"    Failed to determine purpose of statement:\n {s}","error")
            continue

        if state['purpose'] == "CREATE TABLE":
            # _log(f"    Create Table Statement found statement:\n {s}","success")
            state['data'] = parse_table_statement(state['raw'])
            _lu.append(data['tables'],state['data'])
            continue

        if state['purpose'] == "CREATE SCHEMA":
            state['data'] = parse_schema_statement(state['raw'])
            _lu.append(data['schemas'],state['data'])
            continue

        data['statements'].append(state)

    data = _parse_data_comments(data)
    return data

def _parse_json_yaml(comment):
    if comment is not None:
        if isinstance(comment,(str)):
            comment = _sql.escape_quoted_chars(comment,True)
            json = _csu.parse.safe_load_json(comment)
            if json:
                return (comment,json)
            else:
                json = _parse_comment_yaml(comment)
                if json:
                    return (comment,json)
    return (comment,{})

def _parse_data_comments(data):
    new_tables = []
    for tb in data['tables']:
        # print(f"tb['comment']:{tb['comment']}")
        tb['comment'],json = _parse_json_yaml(tb['comment'])
        tb['orig_comment'] = tb['comment']
        tb['comment'] = json
        # tb = {**tb,**json}
        new_tables.append(tb)
        # if tb['comment'] is not None:
        #     tb['comment'] = _sql.escape_quoted_chars(tb['comment'],True)
        #     json = _parse_comment_yaml(tb['comment'])
        #     if json:
        #         tb = {**tb,**json}

        if 'columns' in tb:
            new_cols = []
            for col in tb['columns']:
                col['comment'],json = _parse_json_yaml(col['comment'])
                col = {**col,**json}
                new_cols.append(col)

                # col['comment']
                # if col['comment'] is not None:
                #     if isinstance(col['comment'],(str)):
                #         col['comment'] = _sql.escape_quoted_chars(col['comment'],True)
                #         json = _csu.parse.safe_load_json(col['comment'])
                #         if json:
                #             col = {**col,**json}
                #             col['json_comment_found'] = True
                #             new_cols.append(col)
                #         else:
                #             json = _parse_comment_yaml(col['comment'])
                #             if json:
                #                 col = {**col,**json}
                #                 col['yaml_comment_found'] = True
                #                 new_cols.append(col)
                # else:
                    # new_cols.append(col)
            tb['columns'] = new_cols
        # new_tables.append(tb)
    data['tables'] = new_tables
    return data

def format_sql_string(sql):
    '''
        Format an SQL string to a consistent indentation.

        ----------

        Arguments
        -------------------------
        `sql` {string}
            The sql string to mod.

        Return {string}
        ----------------------
        The formatted SQL

        Meta
        ----------
        `author`: Colemen Atwood
        `created`: 03-21-2022 12:38:32
        `version`: 1.0
        `method_name`: format_sql_string
        @xxx [03-21-2022 12:38:41]: documentation for format_sql_string
    '''
    if _f.exists(sql):
        sql = _f.readr(sql)

    sql = _csu.strip_empty_lines(sql)

    raw = sql
    # raw = _re.sub(r'^[\n]*','',raw)
    # statements = _sqlparse.split(raw)
    statements = get_statements(raw)
    new_contents = []
    for state in statements:
        state = state.value
        # print(f"state: {state}")
        # print(type(state))
        state = _re.sub(r'^[\s\n]*','',state)
        new_state = _sqlparse.format(state, reindent=False, keyword_case='upper')
        new_contents.append(new_state)
        
    if len(new_contents) > 0:
        result = "".join(new_contents)

        result = _re.sub(r"\n\s+","\n ",result)
        # # result = _re.sub(r",\n","__NEW_LINE__",result)
        result = _re.sub(r"DEFAULT\s*CHARSET=[a-zA-Z_0-9]*","",result)
        result = _re.sub(r"ENGINE=[a-zA-Z]*","",result)
        result = _re.sub(r"COLLATE[=\s][a-zA-Z0-9_]*","",result)
        result = _re.sub(r"AUTO_INCREMENT=[0-9]*","",result)
        
        result = result.replace(" DEFAULT NULL"," NULL DEFAULT NULL")
        # # result = result.replace("__NEW_LINE__",",\n")
        # result = result.replace("  "," ")
        
        result = _re.sub(r"[ ]{2,}(?!\n)"," ",result)

        return result
        return new_contents
    return False



# def parse_new(sql):
#     # print("        sql_parse.parse")
#     if _f.exists(sql):
#         # print(f"    Reading SQL file: {sql}")
#         sql = _f.readr(sql)

#     if len(sql)== 0:
#         return None

#     sql = _csu.strip_empty_lines(sql)
#     raw_statements = get_statements(sql)
#     # print(f"raw_statements: {len(raw_statements)}")
#     data = {
#         "schemas":[],
#         "tables":[],
#         "statements":[],
#     }

#     for s in raw_statements:
#         s = s.value
#         # print(f"s: {s}")
#         state = {
#             "raw":s,
#             "purpose":determine_statement_purpose(s),
#             "data":None,
#         }

#         if state['purpose'] is None:
#             _log(f"    Failed to determine purpose of statement:\n {s}","error")
#             continue

#         if state['purpose'] == "CREATE TABLE":
#             # _log(f"    Create Table Statement found statement:\n {s}","success")
#             table = _table_statement.TableStatement(state['raw'])
#             table.parse()
#             state['data'] = table.summary
#             # state['data'] = parse_table_statement(state['raw'])
#             _lu.append(data['tables'],state['data'])
#             continue

#         if state['purpose'] == "CREATE SCHEMA":
#             state['data'] = parse_schema_statement(state['raw'])
#             _lu.append(data['schemas'],state['data'])
#             continue

#         data['statements'].append(state)

#     data = _parse_data_comments(data)
#     return data

