#   This file is part of nexdatas - Tango Server for NeXus data writer
#
#    Copyright (C) 2012-2018 DESY, Jan Kotanski <jkotan@mail.desy.de>
#
#    nexdatas is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    nexdatas is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with nexdatas.  If not, see <http://www.gnu.org/licenses/>.
#

""" Command-line tool for showing meta data from Nexus Files"""

import sys
import argparse
import json
import uuid
import os
import stat
import re
import time
import pytz
import datetime
import pwd
import grp
import fnmatch
import yaml

from .nxsparser import TableTools
from .nxsfileparser import (NXSFileParser, numpyEncoder)
from .nxsargparser import (Runner, NXSArgParser, ErrorException)
from . import filewriter
from .ontology import id_techniques, nexus_panet

WRITERS = {}
try:
    from . import h5pywriter
    WRITERS["h5py"] = h5pywriter
except Exception:
    pass

try:
    from . import h5cppwriter
    WRITERS["h5cpp"] = h5cppwriter
except Exception:
    pass


class General(Runner):

    """ General runner"""

    #: (:obj:`str`) command description
    description = "show general information for the nexus file"
    #: (:obj:`str`) command epilog
    epilog = "" \
        + " examples:\n" \
        + "       nxsfileinfo general /user/data/myfile.nxs\n" \
        + "\n"

    def create(self):
        """ creates parser

        """
        self._parser.add_argument(
            "--h5py", action="store_true",
            default=False, dest="h5py",
            help="use h5py module as a nexus reader")
        self._parser.add_argument(
            "--h5cpp", action="store_true",
            default=False, dest="h5cpp",
            help="use h5cpp module as a nexus reader")

    def postauto(self):
        """ parser creator after autocomplete run """
        self._parser.add_argument(
            'args', metavar='nexus_file', type=str, nargs=1,
            help='new nexus file name')

    def run(self, options):
        """ the main program function

        :param options: parser options
        :type options: :class:`argparse.Namespace`
        :returns: output information
        :rtype: :obj:`str`
        """
        if options.h5cpp:
            writer = "h5cpp"
        elif options.h5py:
            writer = "h5py"
        elif "h5cpp" in WRITERS.keys():
            writer = "h5cpp"
        else:
            writer = "h5py"
        if (options.h5py and options.h5cpp) or \
           writer not in WRITERS.keys():
            sys.stderr.write("nxsfileinfo: Writer '%s' cannot be opened\n"
                             % writer)
            sys.stderr.flush()
            self._parser.print_help()
            sys.exit(255)
        wrmodule = WRITERS[writer.lower()]
        try:
            fl = filewriter.open_file(
                options.args[0], readonly=True,
                writer=wrmodule)
        except Exception:
            sys.stderr.write("nxsfileinfo: File '%s' cannot be opened\n"
                             % options.args[0])
            sys.stderr.flush()
            self._parser.print_help()
            sys.exit(255)

        root = fl.root()
        self.show(root)
        fl.close()

    @classmethod
    def parseentry(cls, entry, description):
        """ parse entry of nexus file

        :param entry: nexus entry node
        :type entry: :class:`filewriter.FTGroup`
        :param description: dict description list
        :type description: :obj:`list` <:obj:`dict` <:obj:`str`, `any` > >
        :return: (key, value) name pair of table headers
        :rtype: [:obj:`str`, :obj:`str`]

        """
        key = "A"
        value = "B"
        at = None
        try:
            at = entry.attributes["NX_class"]
        except Exception:
            pass
        if at and filewriter.first(at.read()) == 'NXentry':
            # description.append(None)
            # value = filewriter.first(value)
            key = "Scan entry:"
            value = entry.name
            # description.append({key: "Scan entry:", value: entry.name})
            # description.append(None)
            try:
                vl = filewriter.first(entry.open("title").read())
                description.append(
                    {key: "Title:", value: vl})
            except Exception:
                sys.stderr.write("nxsfileinfo: title cannot be found\n")
                sys.stderr.flush()
            try:
                vl = filewriter.first(
                    entry.open("experiment_identifier").read())
                description.append(
                    {key: "Experiment identifier:",
                     value: vl})
            except Exception:
                sys.stderr.write(
                    "nxsfileinfo: experiment identifier cannot be found\n")
                sys.stderr.flush()
            for ins in entry:
                if isinstance(ins, filewriter.FTGroup):
                    iat = ins.attributes["NX_class"]
                    if iat and filewriter.first(iat.read()) == 'NXinstrument':
                        try:
                            vl = filewriter.first(ins.open("name").read())
                            description.append({
                                key: "Instrument name:",
                                value: vl})
                        except Exception:
                            sys.stderr.write(
                                "nxsfileinfo: instrument name cannot "
                                "be found\n")
                            sys.stderr.flush()
                        try:
                            vl = filewriter.first(
                                ins.open("name").attributes[
                                    "short_name"].read())
                            description.append({
                                key: "Instrument short name:",
                                value: vl
                            })
                        except Exception:
                            sys.stderr.write(
                                "nxsfileinfo: instrument short name cannot"
                                " be found\n")
                            sys.stderr.flush()

                        for sr in ins:
                            if isinstance(sr, filewriter.FTGroup):
                                sat = sr.attributes["NX_class"]
                                if sat and filewriter.first(sat.read()) \
                                   == 'NXsource':
                                    try:
                                        vl = filewriter.first(
                                            sr.open("name").read())
                                        description.append({
                                            key: "Source name:",
                                            value: vl})
                                    except Exception:
                                        sys.stderr.write(
                                            "nxsfileinfo: source name"
                                            " cannot be found\n")
                                        sys.stderr.flush()
                                    try:
                                        vl = filewriter.first(
                                            sr.open("name").attributes[
                                                "short_name"].read())
                                        description.append({
                                            key: "Source short name:",
                                            value: vl})
                                    except Exception:
                                        sys.stderr.write(
                                            "nxsfileinfo: source short name"
                                            " cannot be found\n")
                                        sys.stderr.flush()
                    elif iat and filewriter.first(iat.read()) == 'NXsample':
                        try:
                            vl = filewriter.first(ins.open("name").read())
                            description.append({
                                key: "Sample name:",
                                value: vl})
                        except Exception:
                            sys.stderr.write(
                                "nxsfileinfo: sample name cannot be found\n")
                            sys.stderr.flush()
                        try:
                            vl = filewriter.first(
                                ins.open("chemical_formula").read())
                            description.append({
                                key: "Sample formula:",
                                value: vl})
                        except Exception:
                            sys.stderr.write(
                                "nxsfileinfo: sample formula cannot"
                                " be found\n")
                            sys.stderr.flush()
            try:
                vl = filewriter.first(entry.open("start_time").read())
                description.append({key: "Start time:", value: vl})
            except Exception:
                sys.stderr.write("nxsfileinfo: start time cannot be found\n")
                sys.stderr.flush()
            try:
                vl = filewriter.first(entry.open("end_time").read())
                description.append({key: "End time:",
                                    value: vl})
            except Exception:
                sys.stderr.write("nxsfileinfo: end time cannot be found\n")
                sys.stderr.flush()
            if "program_name" in entry.names():
                pn = entry.open("program_name")
                pname = filewriter.first(pn.read())
                attr = pn.attributes
                names = [att.name for att in attr]
                if "scan_command" in names:
                    scommand = filewriter.first(attr["scan_command"].read())
                    pname = "%s (%s)" % (pname, scommand)
                description.append({key: "Program:", value: pname})
        return [key, value]

    def show(self, root):
        """ show general informations

        :param root: nexus file root
        :type root: class:`filewriter.FTGroup`
        """

        description = []

        attr = root.attributes

        names = [at.name for at in attr]
        fname = filewriter.first(
            (attr["file_name"].read()
             if "file_name" in names else " ") or " ")
        title = "File name: '%s'" % fname

        print("")
        for en in root:
            description = []
            headers = self.parseentry(en, description)
            ttools = TableTools(description)
            ttools.title = title
            ttools.headers = headers
            rstdescription = ttools.generateList()
            title = ""
            print("\n".join(rstdescription).strip())
            print("")


class BeamtimeLoader(object):

    btmdmap = {
        "principalInvestigator": "pi.email",
        # "pid": "beamtimeId",   # ?? is not unique for dataset
        "owner": "leader.lastname",
        "contactEmail": "pi.email",
        "sourceFolder": "corePath",

        "endTime": "eventEnd",    # ?? should be endTime for dataset
        "ownerEmail": "leader.email",
        "description": "title",   # ?? should be from dataset
        "createdAt": "generated",  # ?? should be automatic
        "updatedAt": "generated",  # ?? should be automatic
        # "proposalId": "proposalId",
        "proposalId": "beamtimeId",
    }

    strcre = {
        "creationLocation": "/DESY/{facility}/{beamlineAlias}",
        "type": "raw",
        "isPublished": False,
        "ownerGroup": "{beamtimeId}-dmgt",
        "accessGroups": ["{beamtimeId}-dmgt",
                         "{beamtimeId}-clbt",
                         "{beamtimeId}-part",
                         "{beamline}dmgt",
                         "{beamline}staff"]
    }

    cre = {
        "creationTime": [],  # ?? startTime for dataset !!!
        "ownerGroup": [],  # ??? !!!

        "sampleId": [],  # ???
        "publisheddataId": [],
        "accessGroups": [],  # ???
        "createdBy": [],  # ???
        "updatedBy": [],  # ???
        "createdAt": [],  # ???
        "updatedAt": [],  # ???
        "isPublished": ["false"],
        "dataFormat": [],
        "scientificMetadata": {},
        "orcidOfOwner": "ORCID of owner https://orcid.org "
        "if available",
        "sourceFolderHost": [],
        "size": [],
        "packedSize": [],
        "numberOfFiles": [],
        "numberOfFilesArchived": [],
        "validationStatus": [],
        "keywords": [],
        "datasetName": [],
        "classification": [],
        "license": [],
        "version": [],
        "techniques": [],
        "instrumentId": [],
        "history": [],
        "datasetlifecycle": [],

    }

    dr = {
        "eventStart": [],
        "beamlineAlias": [],
        "leader": [],
        "onlineAnalysis": [],
        "pi.*": [],
        "applicant.*": [],
        "proposalType": [],
        "users": [],
    }

    copymap = {
        "endTime": "scientificMetadata.end_time.value",
        "creationTime": "scientificMetadata.end_time.value",
        "description": "scientificMetadata.title.value",
        "scientificMetadata.ScanCommand":
            "scientificMetadata.program_name.scan_command",
    }

    def __init__(self, options):
        """ loader constructor

        :param options: parser options
        :type options: :class:`argparse.Namespace`
        """
        self.__pid = options.pid
        self.__pap = options.pap
        self.__relpath = options.relpath
        self.__ownergroup = options.ownergroup
        self.__accessgroups = None
        if options.accessgroups is not None:
            self.__accessgroups = options.accessgroups.split(",")
        dct = {}
        if options.beamtimemeta:
            with open(options.beamtimemeta, "r") as fl:
                # jstr = fl.read()
                # # print(jstr)
                dct = json.load(fl)
        self.__btmeta = dct
        dct = {}
        if options.scientificmeta:
            with open(options.scientificmeta, "r") as fl:
                jstr = fl.read()
                # print(jstr)
                try:
                    dct = json.loads(jstr)
                except Exception:
                    if jstr:
                        nan = float('nan')    # noqa: F841
                        dct = eval(jstr)
                        # mdflatten(dstr, [], dct)
        if 'scientificMetadata' in dct.keys():
            self.__scmeta = dct['scientificMetadata']
        else:
            self.__scmeta = dct
        self.__metadata = {}

    def run(self):
        """ runner for DESY beamtime file parser

        :param options: parser options
        :type options: :class:`argparse.Namespace`
        :returns: metadata dictionary
        :rtype: :obj:`dict` <:obj:`str`, `any`>
        """

        if self.__pap:
            self.btmdmap["proposalId"] = "proposalId"
        if self.__btmeta:
            for sc, ds in self.btmdmap.items():
                sds = ds.split(".")
                md = self.__btmeta
                for sd in sds:
                    if sd in md:
                        md = md[sd]
                    else:
                        print("%s cannot be found" % ds)
                        break
                else:
                    self.__metadata[sc] = md
            for sc, vl in self.strcre.items():
                if isinstance(vl, list):
                    self.__metadata[sc] = [
                        (vv.format(**self.__btmeta)
                         if hasattr(vv, "format") else vv)
                        for vv in vl
                    ]
                else:
                    if hasattr(vl, "format"):
                        self.__metadata[sc] = vl.format(**self.__btmeta)
                    else:
                        self.__metadata[sc] = vl
        if self.__relpath and "sourceFolder" in self.__metadata:
            self.__metadata["sourceFolder"] = \
                os.path.join(self.__metadata["sourceFolder"], self.__relpath)
        if self.__scmeta or self.__btmeta:
            self.__metadata["scientificMetadata"] = {}
        if self.__scmeta:
            self.__metadata["scientificMetadata"].update(self.__scmeta)
        if self.__btmeta and \
           "beamtimeId" not in self.__metadata["scientificMetadata"]:
            self.__metadata["scientificMetadata"]["beamtimeId"] = \
                self.__btmeta["beamtimeId"]
        if self.__btmeta and \
           "DOOR_proposalId" not in self.__metadata["scientificMetadata"]:
            self.__metadata["scientificMetadata"]["DOOR_proposalId"] = \
                self.__btmeta["proposalId"]
        if self.__pid:
            self.__metadata["pid"] = self.__pid
        if self.__ownergroup:
            self.__metadata["ownerGroup"] = self.__ownergroup
        if self.__accessgroups is not None:
            self.__metadata["accessGroups"] = self.__accessgroups
        # print(self.__metadata)
        return self.__metadata

    def merge(self, metadata):
        """ update metadata with dictionary

        :param metadata: metadata dictionary to merge in
        :type metadata: :obj:`dict` <:obj:`str`, `any`>
        :returns: metadata dictionary
        :rtype: :obj:`dict` <:obj:`str`, `any`>
        """
        if not self.__metadata:
            return metadata
        elif not metadata:
            return metadata
        return dict(self._mergedict(metadata, self.__metadata))

    def overwrite(self, metadata, cmap=None):
        """ overwrite metadata with dictionary

        :param metadata: metadata dictionary to merge in
        :type metadata: :obj:`dict` <:obj:`str`, `any`>
        :param cmap: overwrite dictionary
        :type cmap: :obj:`dict` <:obj:`str`, :obj:`str`>
        :returns: metadata dictionary
        :rtype: :obj:`dict` <:obj:`str`, `any`>
        """
        if cmap is None:
            cmap = self.copymap
        if metadata:
            for ts, vs in self.copymap.items():
                vls = vs.split(".")
                md = metadata
                for vl in vls:
                    if vl in md:
                        md = md[vl]
                    else:
                        break
                else:
                    tgs = ts.split(".")
                    td = metadata
                    parent = None
                    for tg in tgs:
                        parent = td
                        if tg in td:
                            td = td[tg]
                        else:
                            td = td[tg] = {}
                    parent[tg] = md
        return metadata

    @classmethod
    def _mergedict(self, dct1, dct2):
        for key in set(dct1) | set(dct2):
            if key in dct1 and key in dct2:
                if isinstance(dct1[key], dict) and isinstance(dct2[key], dict):
                    yield (key, dict(self._mergedict(dct1[key], dct2[key])))
                else:
                    yield (key, dct2[key])
            elif key in dct1:
                yield (key, dct1[key])
            else:
                yield (key, dct2[key])

    def update_pid(self, metadata, filename=None, puuid=False, pfname=False,
                   beamtimeid=None):
        """ update pid metadata with dictionary

        :param metadata: metadata dictionary to merge in
        :type metadata: :obj:`dict` <:obj:`str`, `any`>
        :param filename: nexus filename
        :type filename: :obj:`str`
        :param puuid: pid with uuid
        :type puuid: :obj:`bool`
        :param pfname: pid with file name
        :type pfname: :obj:`bool`
        :returns: metadata dictionary
        :rtype: :obj:`dict` <:obj:`str`, `any`>
        """
        metadata = metadata or {}
        if "pid" not in metadata:
            beamtimeid = beamtimeid or ""
            if not beamtimeid and "scientificMetadata" in metadata \
               and "beamtimeId" in metadata["scientificMetadata"]:
                beamtimeid = metadata["scientificMetadata"]["beamtimeId"]
            scanid = ""
            fsscanid = ""
            fiscanid = None
            fname = ""
            if filename:
                fdir, fname = os.path.split(filename)
                fname, fext = os.path.splitext(fname)
                lfl = fname.split("_")
                fsscanid = lfl[-1]
                res = re.search(r'\d+$', fsscanid)
                fiscanid = int(res.group()) if res else None
            esscanid = ""
            eiscanid = None
            lfl = None
            if "scientificMetadata" in metadata and \
               "name" in metadata["scientificMetadata"] and \
               metadata["scientificMetadata"]["name"]:
                lfl = metadata["scientificMetadata"]["name"].split("_")
                esscanid = lfl[-1]
                res = re.search(r'\d+$', esscanid)
                eiscanid = int(res.group()) if res else None
                # if eiscanid:
                #     print("WWWW:", eiscanid)
            if not pfname and fname and fiscanid is not None:
                scanid = fname
            elif not pfname and fname and eiscanid is not None:
                scanid = "%s_%s" % (fname, eiscanid)
            elif not pfname and fname:
                scanid = fname
            elif fiscanid is not None:
                scanid = str(fiscanid)
            elif eiscanid is not None:
                scanid = str(eiscanid)
            elif fsscanid and esscanid and esscanid == fsscanid:
                scanid = esscanid
            elif fsscanid and esscanid and esscanid != fsscanid:
                scanid = "%s_%s" % (fsscanid, esscanid)
            elif fsscanid:
                scanid = fsscanid
            else:
                scanid = esscanid
            if beamtimeid and scanid:
                if puuid:
                    metadata["pid"] = "%s/%s/%s" % \
                        (beamtimeid, scanid, str(uuid.uuid4()))
                else:
                    metadata["pid"] = "%s/%s" % \
                        (beamtimeid, scanid)

        if "datasetName" not in metadata and "pid" in metadata:
            spid = metadata["pid"].split("/")
            if len(spid) > 1:
                metadata["datasetName"] = spid[1]
            else:
                metadata["datasetName"] = metadata["pid"]
        return metadata

    def update_techniques(self, metadata, techniques=None):
        """ update techniques

        :param metadata: metadata dictionary to merge in
        :type metadata: :obj:`dict` <:obj:`str`, `any`>
        :param techniques: a list of techniques splitted by comma
        :type techniques: :obj:`str`
        :returns: metadata dictionary
        :rtype: :obj:`dict` <:obj:`str`, `any`>
        """
        if techniques:
            metadata["techniques"] = \
                self.generate_techniques(techniques.split(","))
        if metadata and "techniques" not in metadata:
            try:
                if "scientificMetadata" in metadata and \
                       "definition" in metadata["scientificMetadata"]:
                    gdefin = metadata["scientificMetadata"]["definition"]
                    if "value" in gdefin:
                        defin = gdefin["value"].split(",")
                        defin = [
                            (df[2:]
                             if (df and str(df).startswith("NX")) else df)
                            for df in defin]
                        if defin:
                            metadata["techniques"] = \
                                self.generate_techniques(defin)
            except Exception as e:
                sys.stderr.write("nxsfileinfo: '%s'\n"
                                 % str(e))

            try:
                if "scientificMetadata" in metadata and \
                       "experiment_description" in \
                       metadata["scientificMetadata"]:
                    gexpdes = \
                        metadata["scientificMetadata"][
                            "experiment_description"]
                    if "value" in gexpdes:
                        try:
                            pids = None
                            tes = None
                            expdes = yaml.safe_load(gexpdes["value"])
                            if "techniques" in expdes:
                                tes = expdes["techniques"]
                            elif "technique" in expdes:
                                tes = [expdes["technique"]]
                            if "techniques_pids" in expdes:
                                pids = expdes["techniques_pids"]
                            elif "technique_pid" in expdes:
                                pids = [expdes["technique_pid"]]
                            if tes is not None and pids is not None:
                                metadata["techniques"] = \
                                    self.generate_techniques(tes, pids)
                            elif tes is not None:
                                metadata["techniques"] = \
                                    self.generate_techniques(tes)
                            elif pids is not None:
                                metadata["techniques"] = \
                                    self.generate_techniques(pids)
                            else:
                                metadata["techniques"] = \
                                    self.generate_techniques(
                                        [gexpdes["value"]])
                        except Exception:
                            metadata["techniques"] = \
                                self.generate_techniques(
                                    [gexpdes["value"]])
            except Exception as e:
                sys.stderr.write("nxsfileinfo: '%s'\n"
                                 % str(e))

        if metadata and "techniques" not in metadata:
            metadata["techniques"] = []
        return metadata

    def generate_techniques(self, techniques, techniques_pids=None):
        """ generate technique dictionary

        :param techniques: a list of techniques splitted by comma
        :type techniques: :obj:`list` <:obj:`str`>
        :param techniques_pids: a list of technique pids splitted by comma
        :type techniques_pids: :obj:`list` <:obj:`str`>
        :returns: technique dictionary
        :rtype: :obj:`dict` <:obj:`str`, `objstr`>
        """
        result = []
        # print(techniques)
        # print(techniques_pids)
        for it, te in enumerate(techniques):
            pid = None
            name = te
            if techniques_pids and len(techniques_pids) > it and \
               techniques_pids[it] is not None:
                pid = techniques_pids[it]
            elif te.startswith("http:/") and te in id_techniques.keys():
                pid = te
                name = id_techniques[pid]
            elif te in nexus_panet.keys():
                pid = nexus_panet[te]
                name = id_techniques[pid]
            elif te.startswith("PaNET"):
                nm = "http://purl.org/pan-science/PaNET/%s" % te
                if nm in id_techniques.keys():
                    pid = nm
                    name = id_techniques[pid]
            if pid:
                result.append({"pid": pid, "name": name})
            else:
                result.append({"name": name})
        # print(result)
        return result


class Metadata(Runner):

    """ Metadata runner"""

    #: (:obj:`str`) command description
    description = "show metadata information for the nexus file"
    #: (:obj:`str`) command epilog
    epilog = "" \
        + " examples:\n" \
        + "       nxsfileinfo metadata /user/data/myfile.nxs\n" \
        + "       nxsfileinfo metadata /user/data/myfile.nxs -p 'Group'\n" \
        + "       nxsfileinfo metadata /user/data/myfile.nxs -s\n" \
        + "       nxsfileinfo metadata /user/data/myfile.nxs " \
        + "-a units,NX_class\n" \
        + "\n"

    def create(self):
        """ creates parser

        """
        self._parser.add_argument(
            "-a", "--attributes",
            help="names of field or group attributes to be show "
            " (separated by commas without spaces). "
            "The default takes all attributes",
            dest="attrs", default=None)
        self._parser.add_argument(
            "-n", "--hidden-attributes",
            help="names of field or group attributes to be hidden "
            " (separated by commas without spaces). "
            "The default: 'nexdatas_source,nexdatas_strategy,units'",
            dest="nattrs", default="nexdatas_source,nexdatas_strategy,units")
        self._parser.add_argument(
            "-v", "--values",
            help="field names of more dimensional datasets"
            " which value should be shown"
            " (separated by commas without spaces)",
            dest="values", default="")
        self._parser.add_argument(
            "-w", "--owner-group",
            default="", dest="ownergroup",
            help="owner group name. Default is {beamtimeid}-dmgt")
        self._parser.add_argument(
            "-c", "--access-groups",
            default=None, dest="accessgroups",
            help="access group names separated by commas. "
            "Default is {beamtimeId}-dmgt,{beamtimeid}-clbt,{beamtimeId}-part,"
            "{beamline}dmgt,{beamline}staff")
        self._parser.add_argument(
            "-g", "--group-postfix",
            help="postfix to be added to NeXus group name. "
            "The default: ''",
            dest="group_postfix", default="")
        self._parser.add_argument(
            "-t", "--entry-classes",
            help="names of entry NX_class to be shown"
            " (separated by commas without spaces)."
            " If name is '' all groups are shown. "
            "The default: 'NXentry'",
            dest="entryclasses", default="NXentry")
        self._parser.add_argument(
            "-e", "--entry-names",
            help="names of entry groups to be shown"
            " (separated by commas without spaces)."
            " If name is '' all groups are shown. "
            "The default: ''",
            dest="entrynames", default="")
        self._parser.add_argument(
            "-q", "--techniques",
            help="names of techniques"
            " (separated by commas without spaces)."
            "The default: ''",
            dest="techniques", default="")
        self._parser.add_argument(
            "-m", "--raw-metadata", action="store_true",
            default=False, dest="rawscientific",
            help="do not store NXentry as scientificMetadata")
        self._parser.add_argument(
            "--oned", action="store_true",
            default=False, dest="oned",
            help="add 1d values to scientificMetadata")
        self._parser.add_argument(
            "-p", "--pid", dest="pid",
            help=("dataset pid"))
        self._parser.add_argument(
            "-i", "--beamtimeid", dest="beamtimeid",
            help=("beamtime id"))
        self._parser.add_argument(
            "-u", "--pid-with-uuid", action="store_true",
            default=False, dest="puuid",
            help=("generate pid with uuid"))
        self._parser.add_argument(
            "-d", "--pid-without-filename", action="store_true",
            default=False, dest="pfname",
            help=("generate pid without file name"))
        self._parser.add_argument(
            "--proposal-as-proposal", action="store_true",
            default=False, dest="pap",
            help=("Store the DESY proposal as the SciCat proposal"))
        self._parser.add_argument(
            "--h5py", action="store_true",
            default=False, dest="h5py",
            help="use h5py module as a nexus reader")
        self._parser.add_argument(
            "--h5cpp", action="store_true",
            default=False, dest="h5cpp",
            help="use h5cpp module as a nexus reader")
        self._parser.add_argument(
            "-x", "--chmod", dest="chmod",
            help=("json metadata file mod bits, e.g. 0o662"))

    def postauto(self):
        """ parser creator after autocomplete run """
        self._parser.add_argument(
            "-b", "--beamtime-meta", dest="beamtimemeta",
            help=("beamtime metadata file"))
        self._parser.add_argument(
            "-s", "--scientific-meta", dest="scientificmeta",
            help=("scientific metadata file"))
        self._parser.add_argument(
            "-o", "--output", dest="output",
            help=("output scicat metadata file"))
        self._parser.add_argument(
            "-r", "--relative-path", dest="relpath",
            help=("relative path to the scan files"))
        self._parser.add_argument(
            'args', metavar='nexus_file', type=str, nargs="*",
            help='new nexus file name')

    def run(self, options):
        """ the main program function

        :param options: parser options
        :type options: :class:`argparse.Namespace`
        :returns: output information
        :rtype: :obj:`str`
        """
        if options.h5cpp:
            writer = "h5cpp"
        elif options.h5py:
            writer = "h5py"
        elif "h5cpp" in WRITERS.keys():
            writer = "h5cpp"
        else:
            writer = "h5py"
        if (options.h5cpp and options.h5py) or writer not in WRITERS.keys():
            sys.stderr.write("nxsfileinfo: Writer '%s' cannot be opened\n"
                             % writer)
            sys.stderr.flush()
            self._parser.print_help()
            sys.exit(255)

        root = None
        if options.args:
            wrmodule = WRITERS[writer.lower()]
            try:
                fl = filewriter.open_file(
                    options.args[0], readonly=True,
                    writer=wrmodule)
            except Exception:
                sys.stderr.write("nxsfileinfo: File '%s' cannot be opened\n"
                                 % options.args[0])
                sys.stderr.flush()
                self._parser.print_help()
                sys.exit(255)

            root = fl.root()
        self.show(root, options)
        if root is not None:
            fl.close()

    @classmethod
    def metadata(cls, root, options):
        """ get metadata from nexus and beamtime file

        :param options: parser options
        :type options: :class:`argparse.Namespace`
        :param root: nexus file root
        :type root: :class:`filewriter.FTGroup`
        :returns: nexus file root metadata
        :rtype: :obj:`str`
        """
        values = []
        attrs = None
        entryclasses = []
        entrynames = []

        if options.values:
            values = options.values.split(',')

        if options.attrs:
            attrs = options.attrs.split(',')
        elif options.attrs is not None:
            attrs = []

        if options.nattrs not in [None, '', "''", '""']:
            nattrs = options.nattrs.split(',')
        else:
            nattrs = []

        if options.entryclasses not in [None, '', "''", '""']:
            entryclasses = options.entryclasses.split(',')

        if options.entrynames not in [None, '', "''", '""']:
            entrynames = options.entrynames.split(',')

        result = None
        nxsparser = None
        if root is not None:
            nxsparser = NXSFileParser(root)
            nxsparser.valuestostore = values
            nxsparser.group_postfix = options.group_postfix
            nxsparser.entryclasses = entryclasses
            nxsparser.entrynames = entrynames
            nxsparser.scientific = not options.rawscientific
            nxsparser.attrs = attrs
            nxsparser.hiddenattrs = nattrs
            if hasattr(options, "oned"):
                nxsparser.oned = options.oned
            nxsparser.parseMeta()

        if nxsparser is None:
            bl = BeamtimeLoader(options)
            result = bl.run()
            result = bl.overwrite(result)
            result = bl.update_pid(
                result, None, options.puuid,
                options.pfname, options.beamtimeid)
            if not options.rawscientific:
                techniques = None
                if hasattr(options, "techniques"):
                    techniques = options.techniques
                result = bl.update_techniques(result, techniques)
        elif nxsparser and nxsparser.description:
            if len(nxsparser.description) == 1:
                desc = nxsparser.description[0]
                if not options.beamtimemeta:
                    try:
                        if "scientificMetadata" in desc \
                           and "experiment_identifier" in \
                           desc["scientificMetadata"] \
                           and "beamtime_filename" in \
                           desc["scientificMetadata"][
                               "experiment_identifier"]:
                            options.beamtimemeta = \
                                desc["scientificMetadata"][
                                    "experiment_identifier"][
                                        "beamtime_filename"]
                    except Exception:
                        pass
                bl = BeamtimeLoader(options)
                bl.run()
                result = bl.merge(desc)
                result = bl.overwrite(result)
                result = bl.update_pid(
                    result, options.args[0], options.puuid,
                    options.pfname, options.beamtimeid)
                if not options.rawscientific:
                    techniques = None
                    if hasattr(options, "techniques"):
                        techniques = options.techniques
                    result = bl.update_techniques(result, techniques)

            else:
                result = []
                for desc in nxsparser.description:
                    if not options.beamtimemeta:
                        try:
                            if "scientificMetadata" in desc \
                               and "experimental_identifier" in \
                               desc["scientificMetadata"] \
                               and "beamtime_filename" in \
                               desc["scientificMetadata"][
                                   "experimental_identifier"]:
                                options.beamtimemeta = \
                                    desc["scientificMetadata"][
                                        "experimental_identifier"][
                                            "beamtime_filename"]
                        except Exception:
                            pass
                    bl = BeamtimeLoader(options)
                    bl.run()
                    rst = bl.merge(desc)
                    rst = bl.overwrite(rst)
                    rst = bl.update_pid(
                        rst, options.args[0], options.puuid,
                        options.pfname, options.beamtimeid)
                    if not options.rawscientific:
                        techniques = None
                        if hasattr(options, "techniques"):
                            techniques = options.techniques
                        rst = bl.update_techniques(rst, techniques)
                    result.append(rst)
        if result is not None:
            return json.dumps(
                result, sort_keys=True, indent=4,
                cls=numpyEncoder)

    def show(self, root, options):
        """ the main function

        :param options: parser options
        :type options: :class:`argparse.Namespace`
        :param root: nexus file root
        :type root: :class:`filewriter.FTGroup`
        """
        try:
            metadata = self.metadata(root, options)
            if metadata:
                if options.output:
                    chmod = None
                    try:
                        chmod = int(options.chmod, 8)
                    except Exception:
                        options.chmod = None

                    if options.chmod:
                        oldmask = os.umask(0)

                        def opener(path, flags):
                            return os.open(path, flags, chmod)

                        try:
                            with open(options.output,
                                      "w", opener=opener) as fl:
                                fl.write(metadata)
                        except Exception:
                            with open(options.output, "w") as fl:
                                fl.write(metadata)
                            os.chmod(options.output, chmod)
                        os.umask(oldmask)
                    else:
                        with open(options.output, "w") as fl:
                            fl.write(metadata)
                else:
                    print(metadata)
        except Exception as e:
            sys.stderr.write("nxsfileinfo: '%s'\n"
                             % str(e))
            sys.stderr.flush()
            self._parser.print_help()
            sys.exit(255)


class OrigDatablock(Runner):

    """ OrigDatablock runner"""

    #: (:obj:`str`) command description
    description = "show description of all scan files"
    #: (:obj:`str`) command epilog
    epilog = "" \
        + " examples:\n" \
        + "       nxsfileinfo origdatablock /user/data/scan_12345\n" \
        + "\n"

    def create(self):
        """ creates parser

        """
        self._parser.add_argument(
            "-p", "--pid", dest="pid",
            help=("dataset pid"))
        self._parser.add_argument(
            "-w", "--owner-group",
            default="", dest="ownergroup",
            help="owner group name. Default is {beamtimeid}-dmgt")
        self._parser.add_argument(
            "-c", "--access-groups",
            default=None, dest="accessgroups",
            help="access group names separated by commas. "
            "Default is {beamtimeId}-dmgt,{beamtimeid}-clbt,{beamtimeId}-part,"
            "{beamline}dmgt,{beamline}staff")
        self._parser.add_argument(
            "-s", "--skip",
            help="filters for files to be skipped (separated by commas "
            "without spaces). Default: ''. E.g. '*.pyc,*~'",
            dest="skip", default="")
        self._parser.add_argument(
            "-a", "--add",
            help="list of filtes to be added (separated by commas "
            "without spaces). Default: ''. E.g. 'scan1.nxs,scan2.nxs'",
            dest="add", default="")
        self._parser.add_argument(
            "-x", "--chmod", dest="chmod",
            help=("json metadata file mod bits, e.g. 0o662"))

    def postauto(self):
        """ parser creator after autocomplete run """
        self._parser.add_argument(
            'args', metavar='scan_name', type=str, nargs='+',
            help='scan name')
        self._parser.add_argument(
            "-o", "--output", dest="output",
            help=("output scicat metadata file"))
        self._parser.add_argument(
            "-r", "--relative-path", dest="relpath",
            help=("relative path to scan files"))

    def run(self, options):
        """ the main program function

        :param options: parser options
        :type options: :class:`argparse.Namespace`
        :returns: output information
        :rtype: :obj:`str`
        """
        self.show(options)

    def isotime(self, tme):
        """ returns iso time string

        :returns: iso time
        :rtype: :obj:`str`
        """
        tzone = time.tzname[0]
        try:
            tz = pytz.timezone(tzone)
        except Exception:
            import tzlocal
            tz = tzlocal.get_localzone()
        fmt = '%Y-%m-%dT%H:%M:%S.%f%z'
        starttime = tz.localize(datetime.datetime.fromtimestamp(tme))
        return str(starttime.strftime(fmt))

    def filterout(self, fpath, filters):
        found = False
        if filters:
            for df in filters:
                found = fnmatch.filter([fpath], df)
                if found:
                    break
        return found

    def datafiles(self, scanpath, scdir, scfiles, relpath, filters=None):
        dtfiles = []
        totsize = 0
        pdc = {'7': 'rwx', '6': 'rw-', '5': 'r-x', '4': 'r--',
               '3': '-wx', '2': '-w-', '1': '--x', '0': '---'}
        if scdir and scanpath:
            scdir = os.path.relpath(scdir, scanpath)
        for fl in scfiles:
            rec = {}

            fpath = os.path.join(scanpath, scdir, fl)
            if self.filterout(fpath, filters):
                continue
            status = os.stat(fpath)
            prm = str(oct(status.st_mode)[-3:])
            isdir = 'd' if stat.S_ISDIR(status.st_mode) else '-'
            islink = 'l' if stat.S_ISLNK(status.st_mode) else isdir
            perm = islink + ''.join(pdc.get(x, x) for x in prm)

            path = os.path.join(scdir, fl)
            if path.startswith("./"):
                path = path[2:]
            if relpath:
                path = os.path.join(relpath, path)
            rec["path"] = os.path.normpath(path)
            rec["size"] = status.st_size
            rec["time"] = self.isotime(status.st_ctime)
            try:
                rec["uid"] = pwd.getpwuid(status.st_uid).pw_name
            except Exception:
                rec["uid"] = status.st_uid
            try:
                rec["gid"] = grp.getgrgid(status.st_gid).gr_name
            except Exception:
                rec["gid"] = status.st_gid
            rec["perm"] = perm
            dtfiles.append(rec)
            totsize += rec["size"]
        return dtfiles, totsize

    def datablock(self, options):
        """ dump scan datablock JSON

        :param options: parser options
        :type options: :class:`argparse.Namespace`
        :returns: output information
        :rtype: :obj:`str`
        """
        skip = None
        add = []
        if options.skip:
            skip = options.skip.split(',')
        if options.add:
            add = options.add.split(',')

        result = {
        }
        dtfiles = []
        scanfiles = []
        scandirs = []
        totsize = 0
        fscandir = None

        for arg in options.args:
            scandir, scanname = os.path.split(os.path.abspath(arg))
            if not fscandir:
                fscandir = fscandir or scandir
                relpath = options.relpath
            else:
                relpath = os.path.relpath(scandir, fscandir)
                if options.relpath:
                    relpath = os.path.join(options.relpath, relpath)
            for (dirpath, dirnames, filenames) in os.walk(scandir):
                scanfiles = [f for f in filenames if f.startswith(scanname)]
                scandirs = [f for f in dirnames if f.startswith(scanname)]
                break
            flist, tsize = self.datafiles(scandir, "", scanfiles,
                                          relpath, skip)
            dtfiles.extend(flist)
            totsize += tsize

            for fl in add:
                if os.path.isfile(fl):
                    ascandir, ascanname = os.path.split(os.path.abspath(fl))
                    flist, tsize = self.datafiles(
                        scandir, ascandir, [ascanname], relpath)
                    dtfiles.extend(flist)
                    totsize += tsize

            for scdir in scandirs:
                for (dirpath, dirnames, filenames) in os.walk(
                        os.path.join(scandir, scdir)):
                    flist, tsize = self.datafiles(
                        scandir, dirpath, filenames, relpath, skip)
                    dtfiles.extend(flist)
                    totsize += tsize
        result["dataFileList"] = dtfiles
        result["size"] = totsize
        if options.ownergroup:
            result["ownerGroup"] = options.ownergroup
        if options.accessgroups is not None:
            accessgroups = options.accessgroups.split(",")
            result["accessGroups"] = accessgroups
        if options.pid:
            result["datasetId"] = options.pid
        if "datasetId" in result \
           and result["datasetId"] and \
           len(result["datasetId"].split("/")) > 1:
            if "ownerGroup" not in result:
                result["ownerGroup"] = "%s-dmgt" % (
                    result["datasetId"].split("/")[1])
            if "accessGroups" not in result:
                result["accessGroups"] = [
                    "%s-clbt" % (
                        result["datasetId"].split("/")[1]),
                    "%s-dmgt" % (
                        result["datasetId"].split("/")[1])]
        return json.dumps(
                result, sort_keys=True, indent=4,
                cls=numpyEncoder)

    def show(self, options):
        """ the main function

        :param options: parser options
        :type options: :class:`argparse.Namespace`
        """
        try:
            metadata = self.datablock(options)
            if metadata:
                if options.output:
                    chmod = None
                    try:
                        chmod = int(options.chmod, 8)
                    except Exception:
                        options.chmod = None

                    if options.chmod:
                        oldmask = os.umask(0)

                        def opener(path, flags):
                            return os.open(path, flags, chmod)
                        try:
                            with open(options.output,
                                      "w", opener=opener) as fl:
                                fl.write(metadata)
                        except Exception:
                            with open(options.output, "w") as fl:
                                fl.write(metadata)
                            os.chmod(options.output, chmod)
                        os.umask(oldmask)
                    else:
                        with open(options.output, "w") as fl:
                            fl.write(metadata)
                else:
                    print(metadata)
        except Exception as e:
            sys.stderr.write("nxsfileinfo: '%s'\n"
                             % str(e))
            sys.stderr.flush()
            self._parser.print_help()
            sys.exit(255)


class Field(Runner):

    """ Field runner"""

    #: (:obj:`str`) command description
    description = "show field information for the nexus file"
    #: (:obj:`str`) command epilog
    epilog = "" \
        + " examples:\n" \
        + "       nxsfileinfo field /user/data/myfile.nxs\n" \
        + "       nxsfileinfo field /user/data/myfile.nxs -g\n" \
        + "       nxsfileinfo field /user/data/myfile.nxs -s\n" \
        + "\n"

    def create(self):
        """ creates parser

        """
        self._parser.add_argument(
            "-c", "--columns",
            help="names of column to be shown (separated by commas "
            "without spaces). The possible names are: "
            "depends_on, dtype, full_path, nexus_path, nexus_type, shape,"
            " source, source_name, source_type, strategy, trans_type, "
            "trans_offset, trans_vector, units, value",
            dest="headers", default="")
        self._parser.add_argument(
            "-f", "--filters",
            help="full_path filters (separated by commas "
            "without spaces). Default: '*'. E.g. '*:NXsample/*'",
            dest="filters", default="")
        self._parser.add_argument(
            "-v", "--values",
            help="field names which value should be stored"
            " (separated by commas "
            "without spaces). Default: depends_on",
            dest="values", default="")
        self._parser.add_argument(
            "-g", "--geometry", action="store_true",
            default=False, dest="geometry",
            help="perform geometry full_path filters, i.e."
            "*:NXtransformations/*,*/depends_on. "
            "It works only when  -f is not defined")
        self._parser.add_argument(
            "-s", "--source", action="store_true",
            default=False, dest="source",
            help="show datasource parameters")
        self._parser.add_argument(
            "--h5py", action="store_true",
            default=False, dest="h5py",
            help="use h5py module as a nexus reader")
        self._parser.add_argument(
            "--h5cpp", action="store_true",
            default=False, dest="h5cpp",
            help="use h5cpp module as a nexus reader")

    def postauto(self):
        """ parser creator after autocomplete run """
        self._parser.add_argument(
            'args', metavar='nexus_file', type=str, nargs=1,
            help='new nexus file name')

    def run(self, options):
        """ the main program function

        :param options: parser options
        :type options: :class:`argparse.Namespace`
        :returns: output information
        :rtype: :obj:`str`
        """
        if options.h5cpp:
            writer = "h5cpp"
        elif options.h5py:
            writer = "h5py"
        elif "h5cpp" in WRITERS.keys():
            writer = "h5cpp"
        else:
            writer = "h5py"
        if (options.h5cpp and options.h5py) or writer not in WRITERS.keys():
            sys.stderr.write("nxsfileinfo: Writer '%s' cannot be opened\n"
                             % writer)
            sys.stderr.flush()
            self._parser.print_help()
            sys.exit(255)
        wrmodule = WRITERS[writer.lower()]
        try:
            fl = filewriter.open_file(
                options.args[0], readonly=True,
                writer=wrmodule)
        except Exception:
            sys.stderr.write("nxsfileinfo: File '%s' cannot be opened\n"
                             % options.args[0])
            sys.stderr.flush()
            self._parser.print_help()
            sys.exit(255)

        root = fl.root()
        self.show(root, options)
        fl.close()

    def show(self, root, options):
        """ the main function

        :param options: parser options
        :type options: :class:`argparse.Namespace`
        :param root: nexus file root
        :type root: class:`filewriter.FTGroup`
        """
        #: (:obj:`list`< :obj:`str`>)   \
        #     parameters which have to exists to be shown
        toshow = None

        #: (:obj:`list`< :obj:`str`>)  full_path filters
        filters = []

        #: (:obj:`list`< :obj:`str`>)  column headers
        headers = ["nexus_path", "source_name", "units",
                   "dtype", "shape", "value"]
        if options.geometry:
            filters = ["*:NXtransformations/*", "*/depends_on"]
            headers = ["nexus_path", "source_name", "units",
                       "trans_type", "trans_vector", "trans_offset",
                       "depends_on"]
        if options.source:
            headers = ["source_name", "nexus_type", "shape", "strategy",
                       "source"]
            toshow = ["source_name"]
        #: (:obj:`list`< :obj:`str`>)  field names which value should be stored
        values = ["depends_on"]

        if options.headers:
            headers = options.headers.split(',')
        if options.filters:
            filters = options.filters.split(',')
        if options.values:
            values = options.values.split(',')

        nxsparser = NXSFileParser(root)
        nxsparser.filters = filters
        nxsparser.valuestostore = values
        nxsparser.parse()

        description = []
        ttools = TableTools(nxsparser.description, toshow)
        ttools.title = "File name: '%s'" % options.args[0]
        ttools.headers = headers
        description.extend(ttools.generateList())
        print("\n".join(description))


def main():
    """ the main program function
    """

    description = "Command-line tool for showing meta data" \
                  + " from Nexus Files"

    epilog = 'For more help:\n  nxsfileinfo <sub-command> -h'
    parser = NXSArgParser(
        description=description, epilog=epilog,
        formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.cmdrunners = [('field', Field),
                         ('general', General),
                         ('metadata', Metadata),
                         ('origdatablock', OrigDatablock),
                         ]
    runners = parser.createSubParsers()

    try:
        options = parser.parse_args()
    except ErrorException as e:
        sys.stderr.write("Error: %s\n" % str(e))
        sys.stderr.flush()
        parser.print_help()
        print("")
        sys.exit(255)

    if options.subparser is None:
        sys.stderr.write(
            "Error: %s\n" % str("too few arguments"))
        sys.stderr.flush()
        parser.print_help()
        print("")
        sys.exit(255)

    result = runners[options.subparser].run(options)
    if result and str(result).strip():
        print(result)


if __name__ == "__main__":
    main()
