#!/usr/bin/env python
#####################################################################################
# (c) Copyright 1998-2019 CERN for the benefit of the LHCb and ATLAS collaborations #
#                                                                                   #
# This software is distributed under the terms of the Apache version 2 licence,     #
# available at http://www.apache.org/licenses/LICENSE-2.0                           #
#                                                                                   #
# In applying this licence, CERN does not waive the privileges and immunities       #
# granted to it by virtue of its status as an Intergovernmental Organization        #
# or submit itself to any jurisdiction.                                             #
#####################################################################################
import base64
import bisect
import json
import os
import re
import shutil
import sys
import time
import zlib
from optparse import OptionParser

try:
    import xml.etree.ElementTree as ET
except ImportError:
    import xml.etree.ElementTree as ET


def cleanXml(xmlFileName):
    """
    Removes xml illegal characters from a file.
    @param xmlFileName: The name of the xml file.
    """
    _illegal_xml_chars_Re = re.compile(
        "[\x00-\x08\x0b\x0c\x0e-\x1f\ud800-\udfff\ufffe\uffff]"
    )

    def hexreplace(match):
        "Return the hex string"
        return "".join(map(hexConvert, match.group()))

    def hexConvert(char):
        return hex(ord(char))

    print(" Trying to repair the xml file")
    print(" Replacing invalid char")
    xmlFile = open(xmlFileName)
    data = xmlFile.read()
    xmlFile.close()
    xmlFile = open(xmlFileName + "temp", "w")
    xmlFile.write(_illegal_xml_chars_Re.sub(hexreplace, data))
    xmlFile.close()
    try:
        tree = ET.parse(xmlFileName + "temp")
        xmlFile = open(xmlFileName, "w")
        xmlFile.write(data)
        xmlFile.close()
        del tree
    except Exception:
        print(" Removing invalid char")
        xmlFile = open(xmlFileName, "w")
        xmlFile.write(_illegal_xml_chars_Re.sub("", data))
        xmlFile.close()


def formatMeasurementText(txt, escape=False, preformat=True):
    """
    Helper to correctly convert CTest measurements to valid HTML
    (with or without special character escaping and in an optional <pre> tag).
    """
    from codecs import encode
    from xml.sax.saxutils import escape as escape_xml

    # replace xml/html problematic chars and get back to utf
    txt = encode(txt, "utf-8", "xmlcharrefreplace").decode("utf-8")
    if escape:
        txt = escape_xml(txt)
    if preformat:
        txt = f"<pre>{txt}</pre>"
    return txt


VALUE_DECODE = {"base64": base64.b64decode}
VALUE_DECOMP = {"gzip": zlib.decompress}


def dropCustomMeasurements(s):
    """
    Remove custom measurements (sections encompassed by
    '<DartMeasurement ...></DartMeasurement>' tags) from the input string and
    return the new value.
    """
    pos = s.find("<DartMeasurement")
    while pos >= 0:
        end_pos = s.find("</DartMeasurement>", pos)
        if end_pos < 0:
            break  # no end tag, better not to drop the section
        s = s[:pos] + s[end_pos + 18 :]  # 18 is the size of the end tag
        pos = s.find("<DartMeasurement")
    return s


# Some regular expression that is going to be used in the next functions
space_Re = re.compile(r"[ ]")
illegal_web_chars_Re = re.compile(r"[<>\'\"#]")
brackets_Re = re.compile(r"[\[\]\{\}\(\)]")


def cleanWebChar(data):
    """Removes HTML/JSON special characters"""
    return illegal_web_chars_Re.sub("", data)


def cleanSpace(data):
    """Replace spaces in a string by "_" """
    return space_Re.sub("_", data)


def cleanBrackets(data):
    """Removes jquery characters"""
    return brackets_Re.sub("", data)


# redefinition of timedelta.total_seconds() because it is not present in the 2.6 version


def total_seconds_replacement(timedelta):
    return timedelta.days * 86400 + timedelta.seconds + timedelta.microseconds / 1000000


class TestOrganizer:
    """A class to produce a ready to use html from tests. It just have to be feed with
    ElementTree element which are Test Element from a Ctest like XML.
    """

    # status translation from cTest xml to the webinterface
    status = {
        "passed": "PASS",
        "failed": "FAIL",
        "skipped": "SKIPPED",
        "notrun": "SKIPPED",
        "error": "ERROR",
        "untested": "UNTESTED",
    }

    def __init__(self, linkreference, fileName):
        """Initialisation of the class
        @param linkreference: link to the main container which is destinated the html. for
                example : "index.html?test=all_results" This link will be completed to have
                a link to each test
        @param fileName: The name of the file where the html will be stored
        """
        self._root = ET.Element(
            "ul", {"linkreference": linkreference, "class": "topGroup"}
        )
        self.tree = ET.ElementTree(self._root)
        self.seeker = self.elementSeeker()
        self.fileName = fileName
        self._groups = {"Statistics": {}, "GroupList": [], "TestList": []}
        self.statsWritten = False

    def classify(self, testElement):
        """Classify takes a testElement and add it to the element tree representing the html.
        The element is first classed by package hierarchy and then by alphabetical order.
        @param testElement: The test ET.element taken from a ctest Like xml
        @return: True if the test has been inserted, and false in the other cases.
        """
        if testElement is None:
            return False
        Name = testElement.find("Name").text.lower()
        adress = Name.split(".")
        entry = self._root

        groups = self._groups
        oldGroups = groups

        index = 0
        j = None
        for j in range(len(adress[:-1])):
            if adress[j] not in groups:
                if j > 0:
                    index = bisect.bisect(
                        oldGroups[adress[j - 1]]["GroupList"], adress[j]
                    )
                    oldGroups[adress[j - 1]]["GroupList"].insert(index, adress[j])
                else:
                    index = bisect.bisect(oldGroups["GroupList"], adress[j])
                    oldGroups["GroupList"].insert(index, adress[j])

                element = self._addsubgroup(entry, adress[j], j, index)
                groups[adress[j]] = {
                    "Element": element,
                    "Statistics": {},
                    "GroupList": [],
                    "TestList": [],
                }
            else:
                element = groups[adress[j]]["Element"]

            entry = element
            oldGroups = groups
            groups = groups[adress[j]]

        self._addtest(entry, testElement, groups["TestList"], j)

        return True

    def getHtml(self):
        """return the html without statistics"""
        return ET.dump(self._root)

    def write(self):
        """Add statistics and write data to the file."""
        self._addStatistics()
        self.tree.write(self.fileName, "utf-8")

    def _addsubgroup(self, father, sonName, fatherLevel, index):
        """Add a  html group element and return it
        @param father: the  "group" father html  element.
        @param sonName: The name of the group to be added.
        @param fatherLevel: the hierarchical level of the father.
        @param index: the index which points where the group has to be placed
                to be in alphabetical order with its brothers
        """

        # find the ul or create it
        ul = self.seeker.findElement({"tag": "ul", "class": "groupContainer"}, father)
        if ul is None:
            ul = ET.Element("ul", {"class": "groupContainer"})
            father.insert(1, ul)
            if fatherLevel != 0:
                ul.set("style", "display:none;")

        element = ET.Element(
            "li",
            {
                "id": cleanBrackets(sonName) + str(fatherLevel),
                "class": "group foldable",
            },
        )
        ul.insert(index, element)
        element.set(
            "linkreference",
            father.get("linkreference")
            + "/"
            + cleanBrackets(sonName)
            + str(fatherLevel),
        )
        subSpan = ET.SubElement(element, "span", {"class": "clickable folded"})
        subImg = ET.SubElement(subSpan, "img", {"src": "images/plus.png"})
        subImg.text = sonName + ": "

        return element

    def _addtest(self, father, testElement, testList, fatherLevel):
        """Add a html test element
        @param father: the  "group" father html  element.
        @param testElement: The test ET.element taken from a ctest Like xml
        @param testList: a list of the father's tests to find the correct position
                where to place the test in alphabetical order.
        """
        container = self.seeker.findElement({"class": "testContainer"}, father, 2)
        if container is None:
            # Make the container
            div = ET.SubElement(father, "div")
            if fatherLevel is not None:
                div.set("style", "display:none;")
            container = ET.SubElement(div, "ul", {"class": "testContainer"})
        # get the testName
        completeTestName = cleanSpace(cleanWebChar(testElement.find("Name").text))
        testName = cleanSpace(
            cleanWebChar(testElement.find("Name").text.split(".")[-1])
        )
        # Prepare the test
        # ,{"index":cleanBrackets(testName.lower())})
        span = ET.Element("span")

        # test
        li = ET.SubElement(span, "li", {"class": "test foldable"})
        span2 = ET.SubElement(li, "span", {"class": "clickable folded"})
        ET.SubElement(span2, "img", {"src": "images/plus.png"})
        ET.SubElement(
            span2, "span", {"id": cleanBrackets(testName), "class": "testid"}
        ).text = testName + ": "

        # direct link
        link = ET.SubElement(
            li,
            "a",
            {
                "class": "permalink",
                "href": father.get("linkreference") + "//" + cleanBrackets(testName),
            },
        )
        ET.SubElement(
            link, "img", {"title": "Direct link to the test", "src": "images/link.png"}
        )

        status = testElement.get("Status")
        if status in self.status:
            status = self.status[status]
        else:
            status = cleanSpace(cleanWebChar(status))

        ET.SubElement(span2, "span", {"class": status}).text = status + " "

        # add status in groups
        groups = self._groups
        adresses = completeTestName.lower().split(".")[0:-1]
        if len(adresses) > 0:
            for adr in adresses:
                if status not in groups[adr]["Statistics"]:
                    groups[adr]["Statistics"][status] = 0
                groups[adr]["Statistics"][status] += 1
                groups = groups[adr]
        # field container
        ul = ET.SubElement(
            li,
            "ul",
            {
                "class": "fieldid",
                "testName": completeTestName,
                "style": "display: none;",
            },
        )

        self._addField(ul, "start_time")
        self._addField(ul, "end_time")

        # find the well position
        index = bisect.bisect(testList, completeTestName.lower())
        testList.insert(index, completeTestName.lower())

        # insert the test
        container.insert(index, span)

        for namedMeasurment in testElement.find("Results").iter("NamedMeasurement"):
            name = namedMeasurment.get("name")
            if name == "Causes":
                cause = namedMeasurment.find("Value")
                if cause is not None:
                    ET.SubElement(span2, "span", {"class": "cause"}).text = (
                        cause.text + " "
                    )
                continue
            fieldName = cleanSpace(cleanWebChar(name)).lower()
            if fieldName in ("end_time", "start_time", "qmtest.cause", "qmtest.target"):
                continue
            self._addField(ul, fieldName)

        self._addField(ul, "stdout")

    def _addField(self, parent, fieldName):
        """Add a html fiel to a test.
        @param parent: The test html element.
        @param fieldName: The name of the field.
        """
        if parent is not None and fieldName is not None:
            li = ET.Element("li", {"class": "field"})
            li.text = fieldName.lower()
            parent.append(li)

    fieldToAvoidList = ["Statistics", "GroupList", "TestList", "Element"]

    def _addStatistics(self):  # ,groupContainer,fatherAdresses):
        """add statistics to the html structure."""
        for group in self._groups.items():
            if group[0] not in self.fieldToAvoidList:
                self._addStatistic(group[1])

        self.statsWritten = True

    def _addStatistic(self, group):
        """intern recursive method to actually write the statistics
        @param group: Group entry in the self._groups dictionnary.
        """
        if not self.statsWritten:
            stats = ET.SubElement(
                self.seeker.findElement({"tag": "span"}, group["Element"]),
                "span",
                {"class": "statistics"},
            )
        else:
            stats = self.seeker.findElement(
                {"tag": "span", "class": "statistics"},
                self.seeker.findElement({"tag": "span"}, group["Element"]),
            )
            if stats is None:
                stats = ET.SubElement(
                    self.seeker.findElement({"tag": "span"}, group["Element"]),
                    "span",
                    {"class": "statistics"},
                )
            else:
                stats.clear()
                stats.set("class", "statistics")

        for stat in group["Statistics"].items():
            ET.SubElement(stats, "span", {"class": stat[0]}).text = (
                "  " + stat[0] + " = " + str(stat[1])
            )

        # process all the subgroups
        for grp in group.items():
            if grp[0] not in self.fieldToAvoidList:
                self._addStatistic(grp[1])

    def parseXml(self, fileName):
        """Method to find the test element in a cTest like xml.
        @param fileName: the name of the file to parse
        """

        def cleanXml(xmlFileName):
            """
            Removes xml illegal characters from a file.
            @param xmlFileName: The name of the xml file.
            """
            _illegal_xml_chars_Re = re.compile(
                "[\x00-\x08\x0b\x0c\x0e-\x1f\ud800-\udfff\ufffe\uffff]"
            )
            xmlFile = open(xmlFileName)
            data = xmlFile.read()
            xmlFile.close()
            xmlFile = open(xmlFileName, "w")
            xmlFile.write(_illegal_xml_chars_Re.sub("", data))
            xmlFile.close()

        try:
            tree = ET.parse(fileName)
        except Exception:
            cleanXml(fileName)
            tree = ET.parse(fileName)
        newdataset = tree.getroot()

        # Look for all the sites ( there is one site per default but
        # as in the specification it can be more, we consider them
        if newdataset.tag == "Site":
            sites = (
                [newdataset] if newdataset.tag == "Site" else newdataset.getchildren()
            )
        for site in sites:
            print("    -> Process the site: " + site.get("BuildStamp"))
            Testing = site.find("Testing")
            # process each test case
            for Test in Testing.findall("Test"):
                self.classify(Test)

    class elementSeeker:
        """Class which permit to search into an element tree structure."""

        def __init__(self):
            self.swicthtests = {
                "text": self._testtext,
                "tag": self._testtag,
                "class": self._testclass,
            }

        def _testtext(self, element, value):
            return element.text is None or element.text != value

        def _testtag(self, element, value):
            return element.tag is None or element.tag != value

        def _testclass(self, element, value):
            return element.get("class") is None or value not in element.get("class")

        def findElement(self, attributeDictionnary, master, depthMax=-1):
            """
            Recursive function to find an element.Use a depth first algorithm.
            @param attributeDictionnary: A dictionnary of attributes that will be tested
            @param master: the element to search into.
            @param depthMax: The maximum depth to search in. A negative value
                correspond to no limit.
            """
            if len(master) == 0 or depthMax == 0:
                return None
            found = False
            for element in master:
                found = True
                for atributes in attributeDictionnary.items():
                    if atributes[0] in self.swicthtests:
                        if self.swicthtests[atributes[0]](element, atributes[1]):
                            found = False
                            break
                    elif (
                        element.get(atributes[0]) is None
                        or element.get(atributes[0]) != atributes[1]
                    ):
                        found = False
                        break
                if found:
                    return element

            for element in master:
                elementFound = self.findElement(
                    attributeDictionnary, element, depthMax - 1
                )
                if elementFound is not None:
                    return elementFound
            return None


def get_cpuinfo():
    """
    Return the CPU information of the machine as a list of dictionaries.

    Only Linux is supported.
    """
    cpuinfo = []
    try:
        current = {}
        for l in open("/proc/cpuinfo"):
            try:
                k, v = map(str.strip, l.split(":", 1))
                if k == "processor":
                    current = {k: v}
                    cpuinfo.append(current)
                elif k == "flags":
                    current[k] = v.split()
                else:
                    current[k] = v
            except ValueError:
                pass  # ignore lines without a ':'
    except OSError:
        pass  # ignore missing /proc/cpuinfo
    return cpuinfo


def updateAnnotations(site, Testing, outputTemp, annotationsList):
    """
    Update the global annotation file with the data of the current test.
    @param site: The Site ET.Element object  where the annotation information is stored.
    @param Testing: The Testing ET.Element object where datetime information is stored.
    @param outputTemp: The folder where the annotation file is saved.
    @param annotationsList: A list containing the Annotation dictionary.
        Used as a pointer to this object to allow modification.

    """

    if annotationsList[0] == {}:
        annotationsList.pop(0)
        # create the global annotation file
        annotations = {
            "hostname": cleanWebChar(site.get("Hostname")),
            "end_time": cleanWebChar(Testing.find("EndDateTime").text),
            "end_ts": int(cleanWebChar(Testing.find("EndTestTime").text or "0")),
            "start_time": cleanWebChar(Testing.find("StartDateTime").text),
            "start_ts": int(cleanWebChar(Testing.find("StartTestTime").text or "0")),
            "uname": cleanWebChar(
                " ".join(
                    [
                        site.get("OSName"),
                        site.get("Name"),
                        site.get("OSRelease"),
                        site.get("OSVersion"),
                        site.get("OSPlatform"),
                    ]
                )
            ),
            "version": [cleanWebChar(site.get("Generator"))],
        }
        if site.get("qmtest.run.userid") is not None:
            annotations["qmtest.run.userid"] = cleanWebChar(
                site.get("qmtest.run.userid")
            )
        if site.get("qmtest.run.username") is not None:
            annotations["qmtest.run.username"] = cleanWebChar(
                site.get("qmtest.run.username")
            )
        if site.get("qmtest.run.command_line") is not None:
            annotations["qmtest.run.command_line"] = [
                cleanWebChar(site.get("qmtest.run.command_line"))
            ]

        # FIXME: we assume the conversion is run on the same machine as the test
        cpuinfo = get_cpuinfo()
        if cpuinfo:
            annotations["cpuinfo"] = [
                f"ncpus: {len(cpuinfo)}",
                "model name: {}".format(cpuinfo[0].get("model name", os.uname()[-1])),
                "flags: {}".format(
                    " ".join(
                        cpuinfo[0].get("flags") or cpuinfo[0].get("Features") or []
                    )
                ),
            ]

    else:
        annotations = annotationsList.pop(0)
        # update the start time
        startTimeOld = annotations["start_ts"]
        startTimeNew = int(Testing.find("StartTestTime").text)
        if startTimeNew < startTimeOld:
            annotations["start_time"] = time.strftime(
                "%b %d %H:%M %Z", time.localtime(startTimeNew)
            )
            annotations["start_ts"] = startTimeNew
        # update the end time
        endTimeOld = annotations["end_ts"]
        endTimeNew = int(Testing.find("EndTestTime").text or "0")
        if endTimeOld < endTimeNew:
            annotations["end_time"] = time.strftime(
                "%b %d %H:%M %Z", time.localtime(endTimeNew)
            )
            annotations["end_ts"] = endTimeNew
        # adding the generator if new
        if site.get("Generator") not in annotations["version"]:
            annotations["version"].append(cleanWebChar(site.get("Generator")))
        # update qmtest attributes
        if (
            "qmtest.run.userid" not in annotations
            and site.get("qmtest.run.userid") is not None
        ):
            annotations["qmtest.run.userid"] = cleanWebChar(
                site.get("qmtest.run.userid")
            )
        if (
            "qmtest.run.username" not in annotations
            and site.get("qmtest.run.username") is not None
        ):
            annotations["qmtest.run.username"] = cleanWebChar(
                site.get("qmtest.run.username")
            )
        # if "qmtest.run.command_line" not in annotations and site.get("qmtest.run.command_line") is not None:
        #   annotations["qmtest.run.command_line"] = site.get("qmtest.run.command_line")
        if site.get("qmtest.run.command_line") is not None:
            if "qmtest.run.command_line" not in annotations:
                annotations["qmtest.run.command_line"] = [
                    cleanWebChar(site.get("qmtest.run.command_line"))
                ]
            else:
                annotations["qmtest.run.command_line"].append(
                    cleanWebChar(site.get("qmtest.run.command_line"))
                )

    annotationsList.append(annotations)
    # write in the json file
    globalAnnotationFile = open(os.path.join(outputTemp, "annotations.json"), "w")
    globalAnnotationFile.write(json.dumps(annotations, sort_keys=True, indent=4))
    globalAnnotationFile.close()


def main():
    # progStartTime = datetime.now()
    # print "program launch at "+progStartTime.strftime("%b %d %H:%M %Z")

    # parse the command line option
    usage = "usage: %prog [options] Convert CTest like XML into a webpage"
    epilog = "Author: Win-Lime Kevin <kevin.win-lime@telecom-bretagne.eu> (22/08/13)."
    parser = OptionParser(usage=usage, epilog=epilog)
    parser.add_option(
        "-i", "--inputFile", action="store", metavar="FILE", help="read the inputFile"
    )
    parser.add_option(
        "-d",
        "--inputDirectory",
        action="store",
        metavar="DIRECTORY",
        help="read all files ended by Test.xml in the input "
        "directory. The default value is the active "
        "directory",
    )
    parser.add_option(
        "-o",
        "--outputDirectory",
        action="store",
        metavar="DIRECTORY",
        help="Write the result to the output directory. The default value is ./html/",
    )

    parser.set_defaults(inputDirectory=".", outputDirectory="html")
    (options, args) = parser.parse_args()

    # verify the input file
    inputs = []
    if options.inputFile is not None:
        if not os.path.isfile(options.inputFile):
            print(" The input file is not a valid file")
            exit(1)
        else:
            inputs.append(options.inputFile)
    # verify the input directory and search in it for Test.xml file.
    # only if there is no filename specified. by default, the program search all the file in its (sub)directirie
    if not inputs:
        for r, _d, f in os.walk(options.inputDirectory):
            for files in f:
                if files.endswith("Test.xml"):
                    inputs.append(os.path.join(r, files))

    # close the program if there is no file found
    if inputs == []:
        print(" No file found \n")
        exit()

    print(
        "Converting *Test.xml files from %s to HTML format in %s"
        % (options.inputDirectory, options.outputDirectory)
    )
    # verify or create the output directory
    output = options.outputDirectory
    if not os.path.isdir(output):
        os.makedirs(output)

    # Copy the project build manifest, if present.
    manifest_file = os.path.join(options.inputDirectory, "config", "manifest.xml")
    if os.path.exists(manifest_file):
        shutil.copy(manifest_file, output)

    # Initialise the annotations
    annotations = [{}]
    # create the summary file
    globalSummaryFile = open(os.path.join(output, "summary.json"), "w")
    globalSummaryFile.write("[]")
    globalSummaryFile.close()

    # Initialisation of the test organizer
    organizer_all = TestOrganizer(
        "index.html?test=all_results", os.path.join(output, "all_results")
    )
    organizer_failed = TestOrganizer(
        "index.html?test=results", os.path.join(output, "results")
    )
    failures_count = {"FAIL": 0, "ERROR": 0}

    # process each file
    for fileName in inputs:
        print("Process the file : " + fileName)
        # read and parse the xml file
        try:
            tree = ET.parse(fileName)
        except Exception:
            cleanXml(fileName)
            tree = ET.parse(fileName)

        newdataset = tree.getroot()

        # Look for all the sites ( there is one site per default but
        # as in the specification it can be more, we consider them
        sites = [newdataset] if newdataset.tag == "Site" else newdataset.getchildren()
        for site in sites:
            # print "    -> Process the site : "+site.get("BuildStamp")
            Testing = site.find("Testing")
            tests = Testing.findall("Test")
            if not tests:
                # no tests in this "site", let's ignore it
                continue

            # create a directory containing the html file for each site
            # os.path.join(output,os.path.split(fileName)[1][:-8]+str(site.get("BuildStamp")))
            outputTemp = output
            if not os.path.isdir(outputTemp):
                os.makedirs(outputTemp)

            updateAnnotations(site, Testing, outputTemp, annotations)
            """ If the generator is CTest add datetime element to the XML"""
            if "ctest" in site.get("Generator"):
                testStartTime = int(Testing.find("StartTestTime").text)
                # for test in Testing.findall("Test"):

            # find the skeleton directory
            skel = os.path.join(os.path.dirname(__file__), "HTMLTestReportSkel")
            if not os.path.isdir(skel):
                print(f"missing directory {skel}, check installation")
                exit(1)

            # Copy the template directory (do not overwrite)
            for dirpath, dirnames, filenames in os.walk(skel):
                destpath = os.path.join(outputTemp, os.path.relpath(dirpath, skel))
                for d in dirnames:
                    if d in (".svn", ".git"):
                        continue
                    d = os.path.join(destpath, d)
                    if not os.path.exists(d):
                        os.mkdir(d)
                for f in filenames:
                    src = os.path.join(dirpath, f)
                    dst = os.path.join(destpath, f)
                    if not os.path.exists(dst):
                        shutil.copy(src, dst)

            # process each test case
            for Test in tests:
                labels = [l.text for l in Test.findall("Labels/Label")]

                Results = Test.find("Results")
                basedir = Test.find("Path")
                if basedir is None or basedir.text is None:
                    basedir = os.curdir
                else:
                    basedir = basedir.text
                """___ create the local summary   ___"""

                summary = {
                    "id": cleanSpace(cleanWebChar(Test.find("Name").text)),
                    # some interface specific field
                    "fields": ["end_time", "start_time", "stdout"],
                    "cause": "",
                }
                status = Test.get("Status")
                summary["outcome"] = TestOrganizer.status.get(
                    status, cleanSpace(cleanWebChar(status))
                )

                if "test-wrapper" in labels and summary["outcome"] == "PASS":
                    # we should not report the results of test wrappers (like
                    # QMTest) if they are successful
                    continue

                organizer_all.classify(Test)
                if summary["outcome"] != "PASS":
                    organizer_failed.classify(Test)
                if summary["outcome"] in failures_count:
                    failures_count[summary["outcome"]] += 1

                # Fill the summary fields
                for NamedMeasurement in Results.iter("NamedMeasurement"):
                    # remove the space character
                    summary["fields"].append(
                        cleanSpace(cleanWebChar(NamedMeasurement.get("name"))).lower()
                    )
                    # or NamedMeasurement.get("name") == "Pass Reason" :
                    if (
                        NamedMeasurement.get("name") == "Causes"
                        and NamedMeasurement.find("Value") is not None
                        and NamedMeasurement.find("Value").text is not None
                    ):
                        summary["cause"] = cleanWebChar(
                            NamedMeasurement.find("Value").text
                        )

                # update the global summary file
                globalSummaryFile = open(os.path.join(outputTemp, "summary.json"))
                globalSummary = json.loads(globalSummaryFile.read())
                globalSummaryFile.close()
                globalSummary.append(summary)
                globalSummaryFile = open(os.path.join(outputTemp, "summary.json"), "w")
                globalSummaryFile.write(
                    json.dumps(globalSummary, sort_keys=True, indent=4)
                )
                globalSummaryFile.close()
                """___ Create and fill the test directory ___"""
                # create the dir
                testCaseDir = os.path.join(outputTemp, summary["id"])
                if not os.path.isdir(testCaseDir):
                    os.mkdir(testCaseDir)
                # write the local summary
                summaryFile = open(os.path.join(testCaseDir, "summary.json"), "w")
                summaryFile.write(json.dumps(summary, sort_keys=True, indent=4))
                summaryFile.close()

                # write the stdout
                try:
                    value = Results.find("Measurement").find("Value")
                    text = value.text or ""
                    if "encoding" in value.attrib:
                        text = VALUE_DECODE[value.attrib["encoding"]](text)
                        if "compression" in value.attrib:
                            text = VALUE_DECOMP[value.attrib["compression"]](text)
                        text = text.decode("utf-8", errors="ignore")
                    text = dropCustomMeasurements(text)
                    text = formatMeasurementText(text, escape=True)
                    # no "Measurement" or no "Value" or no text
                except AttributeError as x:
                    print(f"WARNING: {summary['id']}: AttributeError: {x}")
                    text = "<i>no stdout</i>"
                except KeyError as x:
                    print(f"WARNING: {summary['id']}: KeyError: {x}")
                    # encoding or compressions unknown, keep original text
                    text = formatMeasurementText(value=text, escape=True)
                with open(os.path.join(testCaseDir, "stdout"), "wb") as stdout:
                    stdout.write(text.encode("utf-8"))

                if "ctest" not in site.get("Generator"):
                    # write the other files
                    for NamedMeasurement in Results.iter("NamedMeasurement"):
                        value = NamedMeasurement.find("Value")
                        cleanName = cleanSpace(
                            cleanWebChar(NamedMeasurement.get("name"))
                        ).lower()
                        NamedMeasurementFile = open(
                            os.path.join(testCaseDir, cleanName), "w"
                        )

                        # Consider the void measurment
                        if value is not None and value.text is not None:
                            NamedMeasurementFile.write(
                                formatMeasurementText(value.text)
                            )
                        else:
                            NamedMeasurementFile.write("<pre></pre>")
                        NamedMeasurementFile.close()
                else:
                    executionTime = float(0)
                    # write the other files
                    for NamedMeasurement in Results.iter("NamedMeasurement"):
                        value = NamedMeasurement.find("Value")
                        cleanName = cleanSpace(
                            cleanWebChar(NamedMeasurement.get("name"))
                        ).lower()
                        if cleanName == "execution_time":
                            executionTime = float(value.text)
                        NamedMeasurementFile = open(
                            os.path.join(testCaseDir, cleanName), "w"
                        )

                        # Consider the void measurment
                        if value is not None and value.text is not None:
                            if cleanName in (
                                "new_output_reference_file",
                                "new_error_reference_file",
                            ):
                                ref = os.path.join(basedir, value.text)
                                if os.path.isfile(ref):
                                    shutil.copyfile(
                                        ref,
                                        os.path.join(
                                            testCaseDir, os.path.basename(ref)
                                        ),
                                    )
                                else:
                                    ref = None
                            else:
                                ref = None
                            if ref:
                                NamedMeasurementFile.write(
                                    '<a href="{0}/{1}">{1}</a>'.format(
                                        summary["id"], os.path.basename(ref)
                                    )
                                )
                            else:
                                NamedMeasurementFile.write(
                                    formatMeasurementText(value.text)
                                )
                        else:
                            NamedMeasurementFile.write("<pre></pre>")
                        NamedMeasurementFile.close()
                    # write the interface specific files
                    # The start_time and the end_time are computed for the cTest generator.
                    # These values are only an aproximation.
                    startTimeFile = open(os.path.join(testCaseDir, "start_time"), "w")
                    startTimeFile.write(
                        time.strftime("%b %d %H:%M %Z", time.localtime(testStartTime))
                    )
                    startTimeFile.close()
                    testEndTime = testStartTime + executionTime
                    endTimeFile = open(os.path.join(testCaseDir, "end_time"), "w")
                    endTimeFile.write(
                        time.strftime("%b %d %H:%M %Z", time.localtime(testEndTime))
                    )
                    endTimeFile.close()
                    testStartTime = testEndTime
            organizer_all.write()
            organizer_failed.write()

    if sum(failures_count.values()):
        print("Some tests failed:")
        print("\n".join("  %s: %s" % i for i in list(failures_count.items())))
        sys.exit(1)

    # print " End of the program"
    # progEndTime = datetime.now()
    # delta = progEndTime - progStartTime
    # print "program ended at "+progEndTime.strftime("%b %d %H:%M %Z")
    # print "Duration : "+str(total_seconds_replacement(delta))+" seconds"


if __name__ == "__main__":
    main()
