import csv
import glob
import json
import re
from datetime import datetime
from os import path, makedirs
from uuid import uuid4

from excel2meta_interface.utils import messages, helpers


class Reference2Interface:
    """
        Reference2Interface
        Convert reference files, generated by e.g. Excel2Reference, to defined json schema files
            as per https://github.com/jacbeekers/metadata-registry-interface-specifications
    """
    start_time = datetime.now()
    # used the create sub directories
    one_on_one_name = "one-on-one"

    def __init__(self, configuration_file="resources/excel2json_config.json"):
        # default values
        self.json_directory = "out/"
        self.json_file_prefix = "generated_"
        self.schema_base_directory = "resources/metadata-registry-interface-specifications"
        self.version = "0.2.0"
        # within schema_directory:
        self.schema_location = "metadata/schemas/interface/" + self.version
        self.result = messages.message["ok"]
        self.data_sources = []
        self.edc_lineage_column_header = [
            "Association",
            "From Connection",
            "To Connection",
            "From Object",
            "To Object",
        ]
        try:
            with open(configuration_file) as config:
                data = json.load(config)
                self.json_directory = data["json_directory"]
                if "json_file_prefix" in data:
                    self.json_file_prefix = data["json_file_prefix"]
                if "schema_base_directory" in data:
                    self.schema_base_directory = data["schema_base_directory"]
                if "version" in data:
                    self.version = data["version"]
                if "schema_location" in data:
                    self.schema_location = data["schema_location"]
                if "jinja_templates" in data:
                    self.jinja_templates = data["jinja_templates"]
                else:
                    self.jinja_templates = "resources/jinja_templates"
                if "lineage_output" in data:
                    self.lineage_output = data["lineage_output"]
                else:
                    self.lineage_output = "out/"
                if "json_file_prefix" in data:
                    self.json_file_prefix = data["json_file_prefix"]
                else:
                    self.json_file_prefix = self.start_time.isoformat(timespec="microseconds").replace(':', '-')
                if self.json_file_prefix == "TIMESTAMP":
                    self.json_file_prefix = self.start_time.isoformat(timespec="microseconds").replace(':', '-')
                self.json_file_prefix += "--"
                if "one_on_ones" in data:
                    self.one_on_ones = data["one_on_ones"]
                else:
                    self.one_on_ones = None

        except FileNotFoundError as e:
            print("Configuration file >" + configuration_file + "< not found.")
            self.result = messages.message["main_config_not_found"]

        self.one_on_patterns = []
        if self.one_on_ones is not None:
            with open(self.one_on_ones) as one_on_ones:
                patterns = json.load(one_on_ones)["patterns"]
            for pattern in patterns:
                regex_zone_pattern = re.compile(pattern["pattern"])
                label = pattern["label"]
                self.one_on_patterns.append(dict(
                      zone_pattern=regex_zone_pattern
                    , label=label
                    , source_edc_resource=pattern["source_edc_resource"]
                    , source_edc_datasource=pattern["source_edc_datasource"]
                    , source_folder=pattern["source_folder"]
                    , target_edc_resource=pattern["target_edc_resource"]
                    , target_edc_datasource=pattern["target_edc_datasource"]
                ))
        self.one_on_one_data_sources = []
        self.helper = helpers.UtilsHelper()

    def process_generated_files(self):
        mapping_files = []
        reference_files = []
        attribute_files = []
        result = messages.message["ok"]

        # First process all REF files as some objects will get their uuid from it
        for file in glob.glob(self.json_directory + "20*--REF_-_*.json"):
            print("Processing reference file >%s<" % file)
            result = self.process_reference_file(file)
            if result["code"] == "OK":
                reference_files.append(file)

        # Generate attribute files for entities
        # It is assumed that reference_files has already been populated so the uuid of the entity can be found
        for file in glob.glob(self.json_directory + "20*--ATTRIB_-_*.json"):
            print("Processing attributes file >%s<" % file)
            result = self.process_attributes_file(file)
            if result["code"] == "OK":
                attribute_files.append(file)

        # Generate the association files
        for file in glob.glob(self.json_directory + "20*--MAP_-_*.json"):
            print("Processing mapping file >%s<" % file)
            result = self.process_mapping_file(file)
            if result["code"] == "OK":
                mapping_files.append(file)

        return result

    def process_attributes_file(self, attributes_file):
        result = messages.message["ok"]
        result_entity, jinja_attributes = self.helper.get_jinja_template(template_directory=self.jinja_templates
                                                                         , template_name="ref_entity_attributes.jinja")
        result_attribute, jinja_attribute = self.helper.get_jinja_template(template_directory=self.jinja_templates
                                                                           , template_name="ref_attribute.jinja")
        if result_attribute["code"] == "OK" and result_entity["code"] == "OK":
            with open(attributes_file, "r") as input:
                sources = json.load(input)
                for source in sources:
                    the_attributes = []
                    for attribute in source["attributes"]:
                        desc = attribute["attribute_description"].replace('"', '\\"')
                        attrib_content = jinja_attribute.render(
                              attribute_uuid=uuid4()
                            , attribute_name=attribute["attribute_name"]
                            , attribute_description=desc
                        )
                        the_attributes.append(json.loads(attrib_content))
                    # fill in the jinja template for attributes
                    the_uuid = self.get_entity_uuid(source["dataset"], data_sources=self.data_sources)
                    content = jinja_attributes.render(
                          attributes=json.dumps(the_attributes)
                        , entity_uuid=the_uuid
                        , version=self.start_time.strftime("%Y.%m.%d")
                    )
                    # print(content)
                    json_content = json.loads(content)
                    # print("content is >%s<" % json_content)
                    file_name = path.join(self.lineage_output, source["dataset"] + "_attributes.json")
                    with open(file_name, "w", encoding="utf8") as out:
                        json.dump(json_content, out, indent=4)
                    # if the data sources is in a one-on-one layer, also generate an attribute json file for the predecessor
                    if source["is_source"] == "True" and self.one_on_ones is not None:
                        for pattern in self.one_on_patterns:
                            # print("Checking for one-on-one", item["label"])
                            # zone_pattern = re.compile(pattern["pattern"])
                            the_zone = pattern["zone_pattern"].search(source["src_zone"])
                            if the_zone is None:
                                continue
                            else:
                                # print("try to jsonize:", content)
                                result = self.process_one_on_one_attributes(the_zone.group(), source
                                                                 , pattern["label"], the_attributes)
                                if result["code"] != "OK":
                                    return result

        else:
            result = messages.message["jinja_template_not_found"]
        return result

    def get_entity_uuid(self, entity, data_sources):
        # look in self.data_sources for the uuid of the entity
        count = 0
        entity_uuid = None
        for source in data_sources:
            # print("looking for entity_uuid in", source)
            if "dataset" in source["data_source"] and source["data_source"]["dataset"] == entity:
                entity_uuid = source["uuid"]
                count += 1
                # print("get_entity_uuid - found #%d entity >%s<. its uuid is >%s<" % (count, entity, source["uuid"]))

        if count == 0:
            print("Entity not found")
        elif count > 1:
            print("ERROR: Entity found more than once")

        return entity_uuid

    def process_mapping_file(self, mapping_file):
        result = messages.message["ok"]
        if "data-sources-" in mapping_file:
            result = self.process_map_data_source_file(mapping_file)
        elif "zones-" in mapping_file:
            result = self.process_map_zone_file(mapping_file)
        else:
            print("%s: This is not the file we are looking for..." % mapping_file)
        return result

    def process_reference_file(self, reference_file):
        result = messages.message["ok"]
        if "data-sources-" in reference_file:
            if "data-sources-source" in reference_file:
                is_source = True
            else:
                is_source = False
            result = self.process_ref_data_source_file(reference_file, is_source)
        elif "zones-" in reference_file:
            result = self.process_ref_zone_file(reference_file)
        else:
            print("%s: This is not the file we are looking for..." % reference_file)
        return result

    def process_ref_data_source_file(self, data_source_file, is_source=True):
        with open(data_source_file, "r") as input:
            sources = json.load(input)
            # print(sources)

        result, jinja_template = self.helper.get_jinja_template(template_directory=self.jinja_templates
                                                                , template_name="ref_data_source.jinja")
        if result["code"] != "OK":
            return result

        for data_source in sources:
            the_uuid = uuid4()
            data_source["description"] = data_source["description"].replace('"',"'")
            self.data_sources.append(dict(uuid=the_uuid, data_source=data_source))
            print("adding source to data_sources list:", data_source)
            # some string contain unescaped quotes with which json.loads generates an error
            # so need to change the description field accordingly
            desc = data_source["description"]
            # if "\"" in desc:
            #    # print("There's a double-quote in the description field.")
            #    desc = data_source["description"].replace('"', '\\"')
            #    # print("Fix: >%s<" % desc)
            content = jinja_template.render(
                entity_type="file"
                , uuid=the_uuid
                , name=data_source["dataset"]
                , version=self.start_time.strftime("%Y.%m.%d")
                , meta_version="0.2.0"
                , description=desc
                , set_id=str(data_source["id"])
                , app_id=data_source["app"]
            )
            if is_source:
                sub_folder = "sources"
                directory = path.join(self.lineage_output, sub_folder, data_source["src_zone"])
            else:
                sub_folder = "targets"
                directory = path.join(self.lineage_output, sub_folder, data_source["tgt_zone"])
            makedirs(directory, exist_ok=True)
            # create entity json
            file_name = path.join(directory, data_source["dataset"] + "_entity.json")
            with open(file_name, "w", encoding="utf8") as out:
                out.write(content)
                # print("Content is >%s<" % content)
            # create a copy in the sub folder source or target
            file_name = path.join(self.lineage_output, data_source["dataset"] + "_entity")
            with open(file_name + ".json", "w", encoding="utf8") as out:
                out.write(content)
            with open(file_name + ".meta", "w", encoding="utf8") as out:
                if is_source:
                    out.write("is_source=True\n")
                    out.write("zone=" + data_source["src_zone"] + "\n")
                else:
                    out.write("is_source=False\n")
                    out.write("zone=" + data_source["tgt_zone"] + "\n")
            # if the data sources is in a one-on-one layer, also generate a dataset json file for the predecessor
            if is_source and self.one_on_ones is not None:
                todo = True
                for pattern in self.one_on_patterns:
                    # print("Checking for one-on-one", item["label"])
                    # zone_pattern = re.compile(pattern["pattern"])
                    the_zone = pattern["zone_pattern"].search(data_source["src_zone"])
                    if the_zone is None:
                        continue
                    else:
                        # print("try to jsonize:", content)
                        result = self.process_one_on_one(the_zone.group(), data_source["dataset"]
                                                         , pattern["label"], json.loads(content))
                        if result["code"] != "OK":
                            return result
                    if todo:
                        todo = False
                        result = self.generate_configuration_file(self.lineage_output, pattern, data_source)

        return result

    def process_one_on_one(self, zone, source, label, original_content):
        """
            Generate the necessary files to generate a one-on-one mapping between a source and target
        :param zone: (not used)
        :param source:
        :param label:
        :param original_content:
        :return:
        """
        # print("Processing one-on-one for source >%s< in zone >%s<, label is >%s<" % (source, zone, label))

        # get the jinja2 template
        result, jinja_template = self.helper.get_jinja_template(template_directory=self.jinja_templates
                                                                , template_name="ref_data_source.jinja")
        if result["code"] != "OK":
            return result

        # print("original_content:", original_content)
        the_uuid = uuid4()
        # Add only if we have all the info. For now: don't
        the_source = dict(is_source="1:1", dataset=source)
        self.one_on_one_data_sources.append(dict(uuid=the_uuid, data_source=the_source))
        content = jinja_template.render(
            entity_type="file"
            , uuid=the_uuid
            , name=label + "-" + source
            , version=self.start_time.strftime("%Y.%m.%d")
            , meta_version="0.2.0"
            , description=original_content["description"]
            , set_id=str(original_content["dataset_id"])
            , app_id=original_content["application_id"]
        )

        directory = path.join(self.lineage_output, self.one_on_one_name)
        makedirs(directory, exist_ok=True)
        file_name = path.join(directory, label + "-" + source + "-1on1_entity")
        with open(file_name + ".json", "w", encoding="utf8") as out:
            out.write(content)

        return result

    def process_one_on_one_attributes(self, zone, source, label, attributes):
        """
         Creates the attribute json for the 1:1 datasource
        :param zone:
        :param data_set:
        :param label:
        :param attributes:
        :return:
        """
        result =  messages.message["ok"]
        result_entity, jinja_attributes = self.helper.get_jinja_template(template_directory=self.jinja_templates
                                                                         , template_name="ref_entity_attributes.jinja")
        if result_entity["code"] != "OK":
            return result_entity

        result_attribute, jinja_attribute = self.helper.get_jinja_template(template_directory=self.jinja_templates
                                                                           , template_name="ref_attribute.jinja")
        if result_attribute["code"] != "OK":
            return result_attribute

        # generate new uuids for the 1:1
        the_attributes = []
        for attribute in attributes:
            # print("attribute:", attribute)
            the_uuid = uuid4()
            content = jinja_attribute.render(
                  attribute_uuid=the_uuid
                , attribute_name=attribute["name"]
                , attribute_description=attribute["description"]
                )
            the_attributes.append(json.loads(content))

        # print("Looking for >%s< in current one_on_one datasources:" % source["dataset"], self.one_on_one_data_sources)
        the_uuid = self.get_entity_uuid(source["dataset"], data_sources=self.one_on_one_data_sources)
        content = jinja_attributes.render(
              attributes=json.dumps(the_attributes)
            , entity_uuid=the_uuid
            , version=self.start_time.strftime("%Y.%m.%d")
        )

        # print("1:1 attribute content:", content)
        json_content = json.loads(content)

        directory = path.join(self.lineage_output, self.one_on_one_name)
        makedirs(directory, exist_ok=True)
        file_name = path.join(directory, label + "-" + source["dataset"] + "-1on1_attributes.json")
        with open(file_name, "w", encoding="utf8") as out:
            json.dump(json_content, out, indent=4)

        return result

    def generate_configuration_file(self, out_directory, pattern, data_source):
        """
            The configuration file can be used to generate 1:1 lineage in EDC using the edc_replication_lineage code
            in https://github.com/jacbeekers/Informatica-EDC-REST-API-Samples
            pip3 install "informatica-edc-rest-api-samples>=0.3.67"
        :param out_directory:
        :param pattern: pattern as defined by oneonones.json
        :param data_source: the data source object, used to retrieve src_zone from
        :return: messages.message structure
        """
        result = messages.message["ok"]

        result, jinja_config = self.helper.get_jinja_template(template_directory=self.jinja_templates
                                                              , template_name="ref_configuration.jinja")
        if result["code"] != "OK":
            return result

        result, jinja_mapping = self.helper.get_jinja_template(template_directory=self.jinja_templates
                                                               , template_name="edc_source_target_mapping.jinja")
        if result["code"] != "OK":
            return result

        # Note: the following is Informatica EDC naming.
        # TODO: Get the EDC resource name from a configuration file
        mapping_content = jinja_mapping.render(
              source_resource_name=pattern["source_edc_resource"]
            , source_datasource=pattern["source_edc_datasource"]
            , source_container=pattern["source_folder"]
            , target_resource_name=pattern["target_edc_resource"]
            , target_datasource=pattern["target_edc_datasource"]
            , target_container=data_source["src_zone"]
        )

        directory = path.join(out_directory, self.one_on_one_name, "config")
        makedirs(directory, exist_ok=True)
        # print("src_zone is >%s<" % data_source["src_zone"])
        for_zone = data_source["src_zone"].split('/')[0] + "_"
        for_zone += path.basename(data_source["src_zone"])
        filename = "edc_mapping_for_" + for_zone + ".json"
        with open(path.join(directory, filename), "w", encoding="utf8") as config:
            config.write(mapping_content)

        # got the jinja2 templates
        config_content = jinja_config.render(
            edc_source_target_mapping=path.join(directory, filename)
        )
        filename = "config_for_" + for_zone + ".json"
        with open(path.join(directory, filename), "w", encoding="utf8") as config:
            config.write(config_content)

        return result

    def process_ref_zone_file(self, zone_file):
        return messages.message["not_implemented"]

    def process_map_data_source_file(self, data_source_file):
        """
            Generate a csv that can be read by EDC once you've set up a lineage resource to read this csv
            The lineage will show which data sources are connected
        :param data_source_file:
        :return:
        """
        with open(data_source_file, "r") as input:
            src_tgt_data_sources = json.load(input)
        directory = path.join(self.lineage_output, self.one_on_one_name, "lineage")
        makedirs(directory, exist_ok=True)
        csv_file_name = path.join(directory, "lineage_for_data_sources" + ".csv")
        with open(csv_file_name, "w", newline="", encoding="utf8") as f_csv_file:
            col_writer = csv.writer(f_csv_file)
            col_writer.writerow(self.edc_lineage_column_header)
            for src_tgt_data_source in src_tgt_data_sources:
                # print("src_tgt_data_source is:", src_tgt_data_source)
                src_path = path.join("edl://", "FileSystem"
                                     , src_tgt_data_source["source"], src_tgt_data_source["source_dataset"])
                tgt_path = path.join("Rock_Anthems_Playlist://", "FileSystem"
                                     , src_tgt_data_source["target"], src_tgt_data_source["target_dataset"])
                col_writer.writerow(["core.DataSetDataFlow", "", "", src_path, tgt_path])

        return messages.message["ok"]

    def process_map_zone_file(self, zone_file):
        """
            Generate a csv that can be read by EDC once you've setup a lineage resource to read this csv
            The lineage will show which zones (folders) are connected
        :param zone_file:
        :return:
        """
        with open(zone_file, "r") as input:
            src_tgt_zones = json.load(input)
        directory = path.join(self.lineage_output, self.one_on_one_name, "lineage")
        makedirs(directory, exist_ok=True)
        csv_file_name = path.join(directory, "lineage_for_zones" + ".csv")
        with open(csv_file_name, "w", newline="", encoding="utf8") as f_csv_file:
            col_writer = csv.writer(f_csv_file)
            col_writer.writerow(self.edc_lineage_column_header)
            for src_tgt_zone in src_tgt_zones:
                # TODO: Get EDC Resource name from input or config
                # print("src_tgt_zone is:", src_tgt_zone)
                src_path = path.join("edl://", "FileSystem", src_tgt_zone["source"])
                tgt_path = path.join("Rock_Anthems_Playlist://", "FileSystem", src_tgt_zone["target"])
                col_writer.writerow(["core.DataSourceDataFlow", "", "", src_path, tgt_path])

        return messages.message["ok"]

    def main(self):
        if self.result["code"] != "OK":
            print("initialization failure:", self.result)
            return self.result
        else:
            rc = self.process_generated_files()
            return rc
