import fnmatch
import hashlib
import logging
import os
import time
from functools import partial
from multiprocessing.dummy import Pool as ThreadPool
from typing import List

from boto3.exceptions import S3UploadFailedError

from taskcat._logger import PrintMsg
from taskcat.exceptions import TaskCatException

LOG = logging.getLogger(__name__)


class S3Sync:
    """Syncronizes local project files with S3 based on checksums.

    Excludes hidden files, unpackaged lambda source and taskcat /ci/ files.
    Uses the Etag as an md5 which introduces the following limitations
        * Uses undocumented etag algorithm for multipart uploads
        * Does not work wil files uploaded in the console that use SSE encryption
        * see
          https://docs.aws.amazon.com/AmazonS3/latest/API/RESTCommonResponseHeaders.html
          for more info
    Does not support buckets with versioning enabled
    """

    # TODO: better exclusions that support path wildcards, eg. "*/.git/*"
    exclude_files = [".*", "*.md"]
    exclude_path_prefixes = [
        "lambda_functions/source/",
        "functions/source/",
        ".",
        "venv/",
        "taskcat_outputs/",
    ]

    exclude_remote_path_prefixes: List[str] = []

    def __init__(self, s3_client, bucket, prefix, path, acl="private"):
        """Syncronizes local file system with an s3 bucket/prefix

        """
        if prefix != "" and not prefix.endswith("/"):
            prefix = prefix + "/"
        self.s3_client = s3_client
        file_list = self._get_local_file_list(path)
        s3_file_list = self._get_s3_file_list(bucket, prefix)
        self._sync(file_list, s3_file_list, bucket, prefix, acl=acl)

    @staticmethod
    def _hash_file(file_path, chunk_size=8 * 1024 * 1024):
        # This is a bit funky because of the way multipart upload etags are done, they
        # are a md5 of the md5's from each part with the number of parts appended
        # credit to hyperknot https://github.com/aws/aws-cli/issues/2585#issue-226758933
        md5s = []

        with open(file_path, "rb") as file_handle:
            while True:
                data = file_handle.read(chunk_size)
                if not data:
                    break
                md5s.append(hashlib.md5(data))  # nosec

        if len(md5s) == 1:
            return '"{}"'.format(md5s[0].hexdigest())

        digests = b"".join(m.digest() for m in md5s)
        digests_md5 = hashlib.md5(digests)  # nosec
        return '"{}-{}"'.format(digests_md5.hexdigest(), len(md5s))

    # TODO: refactor
    def _get_local_file_list(
        self, path, include_checksums=True
    ):  # pylint: disable=too-many-locals
        file_list = {}
        # get absolute local path
        path = os.path.abspath(os.path.expanduser(path))
        # recurse through directories
        for root, _, files in os.walk(path):
            relpath = os.path.relpath(root, path) + "/"
            exclude_path = False
            # relative path should be blank if there are no sub directories
            if relpath == "./":
                relpath = ""
            # exclude defined paths
            for prefix in S3Sync.exclude_path_prefixes:
                if relpath.startswith(prefix):
                    exclude_path = True
                    break
            if not exclude_path:
                file_list.update(
                    self._iterate_files(files, root, include_checksums, relpath)
                )
        return file_list

    def _iterate_files(self, files, root, include_checksums, relpath):
        file_list = {}
        for file in files:
            exclude = False
            # exclude defined filename patterns
            for pattern in S3Sync.exclude_files:
                if fnmatch.fnmatch(file, pattern):
                    exclude = True
                    break
            if not exclude:
                full_path = root + "/" + file
                if include_checksums:
                    # get checksum
                    checksum = self._hash_file(full_path)
                else:
                    checksum = ""
                file_list[relpath + file] = [full_path, checksum]
        return file_list

    def _get_s3_file_list(self, bucket, prefix):
        objects = {}
        is_paginated = True
        continuation_token = None
        # While there are more results, fetch them from S3
        while is_paginated:
            # if there's no token, this is the initial list_objects call
            if not continuation_token:
                resp = self.s3_client.list_objects_v2(Bucket=bucket, Prefix=prefix)
            # this is a query to get additional pages, add continuation token to get
            # next page
            else:
                resp = self.s3_client.list_objects_v2(
                    Bucket=bucket, Prefix=prefix, ContinuationToken=continuation_token
                )
            if "Contents" in resp:
                for file in resp["Contents"]:
                    # strip the prefix from the path
                    relpath = file["Key"][len(prefix) :]
                    objects[relpath] = file["ETag"]
            if "NextContinuationToken" in resp.keys():
                continuation_token = resp["NextContinuationToken"]
            # If there's no toke in the response we've fetched all the objects
            else:
                is_paginated = False
        return objects

    @staticmethod
    def _exclude_remote(path):
        keep = False
        for exclude in S3Sync.exclude_remote_path_prefixes:
            if path.startswith(exclude):
                keep = True
                break
        return keep

    # TODO: refactor
    def _sync(
        self, local_list, s3_list, bucket, prefix, acl, threads=16
    ):  # pylint: disable=too-many-locals
        # determine which files to remove from S3
        remove_from_s3 = []
        for s3_file in s3_list.keys():
            if s3_file not in local_list.keys() and not self._exclude_remote(s3_file):
                LOG.info(
                    f"s3://{bucket}/{prefix + prefix + s3_file}",
                    extra={"nametag": PrintMsg.S3DELETE},
                )
                remove_from_s3.append({"Key": prefix + s3_file})
        # deleting objects, max 1k objects per s3 delete_objects call
        for objects in [
            remove_from_s3[i : i + 1000] for i in range(0, len(remove_from_s3), 1000)
        ]:
            response = self.s3_client.delete_objects(
                Bucket=bucket, Delete={"Objects": objects}
            )
            if "Errors" in response.keys():
                for error in response["Errors"]:
                    LOG.error("S3 delete error: %s" % str(error))
                raise TaskCatException("Failed to delete one or more files from S3")
        # build list of files to upload
        upload_to_s3 = []
        for local_file in local_list:
            upload = False
            # If file is not present in S3
            if local_file not in s3_list.keys():
                upload = True
            # If checksum is different
            elif local_list[local_file][1] != s3_list[local_file]:
                upload = True
            if upload:
                absolute_path = local_list[local_file][0]
                s3_path = local_file
                upload_to_s3.append([absolute_path, bucket, s3_path])
        # multithread the uploading of files
        pool = ThreadPool(threads)
        func = partial(
            self._s3_upload_file, prefix=prefix, s3_client=self.s3_client, acl=acl
        )
        pool.map(func, upload_to_s3)
        pool.close()
        pool.join()

    @staticmethod
    def _s3_upload_file(paths, prefix, s3_client, acl):
        local_filename, bucket, s3_path = paths
        retry = 0
        # backoff and retry
        while retry < 5:
            LOG.info(
                f"s3://{bucket}/{prefix + s3_path}", extra={"nametag": PrintMsg.S3}
            )
            try:
                s3_client.upload_file(
                    local_filename, bucket, prefix + s3_path, ExtraArgs={"ACL": acl}
                )
                break
            except Exception as e:  # pylint: disable=broad-except
                retry += 1
                LOG.error("S3 upload error: %s" % e)
                # give up if we've exhausted retries, or if the error is not-retryable
                # ie. AccessDenied
                if retry == 5 or (
                    isinstance(e, S3UploadFailedError) and "(AccessDenied)" in str(e)
                ):
                    raise TaskCatException("Failed to upload to S3")
                time.sleep(retry * 2)
