#!/usr/bin/python3

import os
import sys
import shutil
import argparse
import json
import logging

import hashget
import hashget.hashdb

import filelock

from hashget.debian import debsubmit
from hashget.exceptions import HashPackageExists
from hashget.utils import kmgt, fix_hpspec, du
from hashget.filepool import DirFilePool


def build_pool(pooldir, webroot):
    pool = DirFilePool(pooldir)
    for path in pool.pool_files():

        f = hashget.file.File(path)

        subpath = pool.hashspec2subpath(f.hashspec)
        linkpath = os.path.join(webroot, subpath)
        print("{} > {}".format(linkpath, path))
        os.makedirs(os.path.dirname(linkpath))
        os.symlink(path, linkpath)


def process_submitted(hashdb, newdir, webroot=None):

    log.debug("process submitted {} {}".format(newdir, webroot))

    for name in os.listdir(newdir):
        """ for each upload dir """
        updir = os.path.join(newdir, name)

        log.debug('process dir {}'.format(updir))

        metafile = os.path.join(updir, '_meta.json')

        try:
            with open(metafile) as f:
                meta = json.load(f)
        except FileNotFoundError:
            log.warning('no metafile {}'.format(metafile))
            shutil.rmtree(updir)
            return

        pkgbasename = meta['files']['package']
        debsig = pkgbasename[:-4]  # strip .deb

        if not hashdb.sig_present('deb', debsig):
            for basename in os.listdir(updir):
                path = os.path.join(updir, basename)
                if path.endswith('.deb'):

                    anchors.clean_list()

                    log.debug('processing package {}'.format(basename))
                    attrs = dict()
                    attrs['uploaded_ip'] = meta['ip']
                    attrs['uploaded_time'] = meta['time']

                    try:
                        hp = debsubmit(hashdb, path, anchors, attrs=attrs)
                        log.debug('submitted {}'.format(hp))
                    except HashPackageExists as e:
                        log.debug("Exists: {}".format(str(e)))
                    else:
                        if webroot:
                            hp.make_anchors(webroot)
                        log.info('Submitted {} from {}'.format(hp, meta['ip']))
        else:
            log.debug('already have debsig {}'.format(debsig))

        shutil.rmtree(updir)


def do_dump(hashdb, basename, project=None, hpspec=None):
    for hp in hashdb.hplist(project=project, hpspec=hpspec):
        print(hp.json())

def do_list(hashdb, project=None, hpspec=None):
    for hp in hashdb.hplist(project=project, hpspec=hpspec):
        print(hp)

def do_get(hashdb, project=None, hpspec=None, attrlist=None):
    for hp in hashdb.hplist(project=project, hpspec=hpspec):
        outlist=list()
        for attr in attrlist:
            if hasattr(hp, attr):
                outlist.append(str(getattr(hp, attr)))
            elif attr in hp.attrs:
                outlist.append(str(hp.attrs[attr]))
        print(' '.join(outlist))

def do_set(hashdb, project=None, hpspec=None, attr=None, value=None):
    for hp in hashdb.hplist(project=project, hpspec=hpspec):
        if hasattr(hp, attr):
            setattr(hp, attr, value)
        else:
            hp.attrs[attr] = value
        hp.fix()
        hp.save()

def do_purge(hashdb, project=None, hpspec=None, webroot=None, full=False):

    purged = 0

    if hpspec is None:
        print("Will not purge with empty hpspec. Use --hp all")
        return
    for hp in hashdb.hplist(project=project, hpspec=hpspec):
        print("Purge", hp)
        hp.purge(webroot=webroot, full=full)
        purged += 1

    if purged:
        print("Purged {} packages".format(purged))
    else:
        print("nothing found for hpspec: {} project: {}".format(hpspec, project))



def do_status(hashdb, project=None, webroot=None):

    for name, hdb in hashdb.items():
        if name == project or project is None:
            print(name, hdb)
            print("  size: {}".format(kmgt(du(hdb.path))))

            last_crawled = None
            first_crawled = None
            total_size = 0
            total_sum_size = 0
            total_indexed_size = 0
            total_packages = 0
            total_anchors = 0
            total_files = 0
            total_noanchor = 0
            total_noanchor_sum_size = 0
            total_noalink = 0
            total_otheralink = 0

            for hp in hdb.packages_iter():
                total_packages += 1

                if last_crawled is None or hp.attrs['crawled_date'] > last_crawled:
                    last_crawled = hp.attrs['crawled_date']

                if first_crawled is None or hp.attrs['crawled_date'] < first_crawled:
                    first_crawled = hp.attrs['crawled_date']

                total_size += hp.attrs['size']
                total_sum_size += hp.attrs['sum_size']
                total_indexed_size += hp.attrs['indexed_size']
                total_anchors += len(hp.anchors)
                total_files += len(hp.files)

                if len(hp.anchors) == 0:
                    total_noanchor += 1
                    total_noanchor_sum_size += hp.attrs['sum_size']

                if webroot:
                    for hspec, a in hp.get_anchors():
                        apath = os.path.join(webroot, a)
                        if not os.path.islink(apath):
                            log.debug("NO LINK {} to {}".format(apath, hp.path))
                            total_noalink += 1
                        else:
                            if os.readlink(apath) != hp.path:
                                log.debug("OTHER LINK {} > {} (not {})".format(
                                    hspec, os.path.basename(os.readlink(apath)), hp))
                                total_otheralink += 1

            print(
                    "  packages: {}\n"
                    "  first crawled: {}\n"
                    "  last_crawled: {}\n"
                    "  files: {}\n"
                    "  anchors: {}\n"
                    "  packages size: {}\n"
                    "  files size: {}\n"
                    "  indexed size: {} ({:.2f}%)\n"
                    "  noanchor packages: {}\n"
                    "  noanchor size: {}\n"
                    "  no anchor link: {}\n"
                    "  bad anchor link: {}\n"
                    .format(total_packages, first_crawled, last_crawled, total_files, total_anchors,
                            kmgt(total_size), kmgt(total_sum_size), kmgt(total_indexed_size),
                            total_indexed_size*100/total_sum_size if total_sum_size else 0,
                            total_noanchor, total_noanchor_sum_size,
                            total_noalink, total_otheralink))


def main():

    global log
    global anchors

    # raise limit for logger filelock
    file_lock_log = logging.getLogger('filelock')
    file_lock_log.setLevel(logging.ERROR)

    def_submitted = '/var/run/takeup/uploads/new'
    def_webroot = os.getenv('HASHGET_WEBROOT', None)

    clean_targets = ['cacheget', 'hashdb', 'all']
    def_pool=None

    """
        Create arguments
    """
    parser = argparse.ArgumentParser(description='HashGet fetcher and deduplicator')

    g = parser.add_argument_group('Information')
    g.add_argument('--status', default=False, action='store_true', help='status for --project (or all projects)')
    g.add_argument('--list', default=False, action='store_true', help='list all contents of --project (or all)')
    g.add_argument('--dump', default=False, action='store_true',
                   help='dump one package by basename (can use with --project or --hp)')

    g = parser.add_argument_group('Commands')
    g.add_argument('--addproject', default=False, action='store_true', help='create -p project')
    g.add_argument('--rmproject', default=False, action='store_true', help='delete -p project')
    g.add_argument('--modify', default=False, action='store_true', help='modify -p project')
    g.add_argument('--get', default=None, nargs='+', metavar='FIELD', help='set field for HP')
    g.add_argument('--set', default=None, nargs=2, metavar=('ATTR','VALUE'), help='set attr for HP')
    g.add_argument('--build', default=None, metavar='WEBROOT', help='build static web HashDB')
    g.add_argument('--verify', default=None, metavar='ACTION', nargs='?', const=True, help='verify package(s). ACTION is report|delete. def: report')
    g.add_argument('--submitted', default=None, metavar='UPLOADS_DIR',
                   help='process submitted packages ({})'.format(def_submitted))
    g.add_argument('--purge', default=False, action='store_true',
                   help='purge package and links. (search/delete links if --full)')
    g.add_argument('--clean', default=None, metavar='TARGET',
                   help='one of: {}'.format(clean_targets))

    g = parser.add_argument_group('Options')
    g.add_argument('--project', '-p', default=None, metavar='NAME', help='Project name')
    g.add_argument('--path', default=None, help='Path to local HashDB directories')
    g.add_argument('--webroot', default=def_webroot, help='path to web root (def: $HASHGET_WEBROOT={})'.
                   format(def_webroot))

    g.add_argument('--pool', default=def_pool, help='build path to pool (if given, pool in web root created)')
    g.add_argument('--really', default=False, action='store_true', help='additonal flag for dangerous operations')
    g.add_argument('--full', default=False, action='store_true', help='for --purge')

    g.add_argument('--storage', default=None, help='use this storage type for hashdb')
    g.add_argument('--pkgtype', default=None, help='use this pkgtype for hashdb')
    g.add_argument('--hp', default=None, metavar='HPSPEC', help='Process only this HashPackage. HPSPEC is URL | basename or sig:SIGTYPE:SIGNATURE')

    g.add_argument('--logfile', default=None, help='log file name')
    g.add_argument('--verbose', '-v', default=False, action='store_true', help='verbose logging')

    args = parser.parse_args()

    log = logging.getLogger('hashget')

    if args.verbose:
        loglevel = logging.DEBUG
    else:
        loglevel = logging.INFO

    logging.basicConfig(level=loglevel, format='%(asctime)s %(message)s', datefmt='%Y-%m-%d %H:%M:%S')

    if args.logfile:
        fh = logging.FileHandler(args.logfile)
        fh.setFormatter(logging.Formatter('%(asctime)s %(message)s', datefmt='%Y-%m-%d %H:%M:%S'))
        log.addHandler(fh)

    hpspec = fix_hpspec(args.hp)
    hashdb = hashget.hashdb.HashDBClient(path=args.path, load=False)
    anchors = hashget.AnchorList()

    if args.list:
        do_list(hashdb=hashdb, project=args.project, hpspec=hpspec)

    if args.get:
        do_get(hashdb=hashdb, project=args.project, hpspec=hpspec, attrlist=args.get)

    if args.set:
        do_set(hashdb=hashdb, project=args.project, hpspec=hpspec, attr=args.set[0], value=args.set[1])


    if args.status:
        do_status(hashdb, args.project, args.webroot)

    if args.dump:
        do_dump(hashdb, args.dump, args.project, hpspec=hpspec)

    if args.addproject:
        print("create project {}".format(args.project))
        hashdb.create_project(args.project)

    if args.rmproject:
        if not args.really:
            print("Not --really ...")
            sys.exit(1)

        print("rm project {}".format(args.project))
        hashdb.remove_project(args.project)

    if args.modify:
        print("modify project {}".format(args.project))
        p = hashdb[args.project]

        if args.storage:
            pfiles = list(p.full_package_files())

            p.storage = args.storage

            for pfile in pfiles:
                hp = p.hpclass.load(path=pfile)
                os.unlink(pfile)
                print(hp)
                p.writehp(hp)


        if args.pkgtype:
            p.read_config()
            p.pkgtype = args.pkgtype
            p.write_config()

        log.info(p)

    if args.build:
        webroot = args.build

        if args.pool:
            print("build pool {} in webroot {}".format(args.pool, args.build))
            build_pool(args.pool, args.build)
        else:
            print("build web HashDB in {}".format(webroot))
            for name, hdb in hashdb.hashdb.items():
                if name == args.project or not args.project:
                    print(name)
                    for p in hdb.packages_iter():
                        print(p)
                        p.make_anchors(webroot)


    if args.submitted:

        if os.getuid():
            lockfilename = os.path.expanduser("~/.hashget-admin.lock")
        else:
            lockfilename = '/var/run/hashget-admin.lock'

        lock = filelock.FileLock(lockfilename)
        try:
            with lock.acquire(timeout=0):
                process_submitted(hashdb, args.submitted, args.webroot)
        except filelock.Timeout:
            log.warning("lockfile {} is already used".format(lockfilename))

    if args.purge:
        do_purge(hashdb=hashdb, project=args.project, hpspec=hpspec, webroot=args.webroot, full=args.full)

    if args.clean:
        if args.clean not in clean_targets:
            log.error("clean target must be one of {}".format(clean_targets))
            sys.exit(1)

        if args.clean in ['cacheget', 'all']:
            cg = hashget.cacheget.CacheGet()
            cg.clean()

        if args.clean in ['hashdb', 'all']:
            hashdb.clean()


    if args.verify:
        action = args.verify or 'report'
        for hp in hashdb.hplist(project=args.project, hpspec=hpspec):
            if hp.verify():
                print("VERIFIED {}".format(hp))
            else:
                print("FAILED VERIFICATION {}".format(hp))
                if action == 'delete':
                    print("DELETE {}".format(hp.path))
                    hp.delete()

main()
