# Code to generate HTML report of historical information. This report generated
# either via the -coll flag, or via -s 'batch.GenerateHistoricalReport <batchid>'

import os
import time
import html
import sys
import logging
import locale
from texttestlib.default.batch import HTMLgen, HTMLcolors, jenkinschanges
from texttestlib import plugins
from collections import OrderedDict
from glob import glob
from pprint import pformat
from datetime import datetime, timedelta
from .batchutils import convertToUrl, getEnvironmentFromRunFiles
HTMLgen.PRINTECHO = 0


def parseState(state):
    '''Separate brief text and host list from succeeded_runs file, e.g.
    21Jan2006 after 7 reruns placerville, placerville
    Date field is not part of state.'''
    hosts = state.split(", ")
    possibleBrief = hosts[0].rsplit(" ", 1)
    if len(possibleBrief) > 1:
        # possibleBrief is actually [brief, host0]
        hosts[0] = possibleBrief[1]
        return possibleBrief[0] + " ", hosts
    else:
        return "", hosts


def compactHostRepr(hosts):
    """Convert list ['a', 'a', 'b'] -> 'a*2, b'"""
    hostCount = dict((host, hosts.count(host)) for host in set(hosts))
    hostCountStrings = \
        (('*'.join([host, str(hostCount[host])])
          if hostCount[host] > 1 else host
          for host in hostCount))
    return ', '.join(hostCountStrings)


def getWeekDay(tag):
    return plugins.weekdays[time.strptime(tag.split("_")[0], "%d%b%Y")[6]]


class ColourFinder:
    def __init__(self, getConfigValue):
        self.getConfigValue = getConfigValue

    def find(self, title):
        colourName = self.getConfigValue("historical_report_colours", title)
        return self.htmlColour(colourName)

    def htmlColour(self, colourName):
        if colourName and not colourName.startswith("#"):
            return getattr(HTMLcolors, colourName.upper())
        else:
            return colourName


def getDisplayText(tag):
    displayText = "_".join(tag.split("_")[1:])
    if displayText:
        return displayText
    else:
        return tag


def makeTestId(version, extraVersion, testName):
    testId = version
    if extraVersion:
        testId += "." + extraVersion
    testId += testName
    return testId


class TitleWithDateStamp:
    def __init__(self, title):
        self.title = title + " (generated by " + os.getenv("USER", os.getenv("USERNAME", "unknown")) + " at "

    def __str__(self):
        return self.title + plugins.localtime(format="%d%b%H:%M") + ")"


class GenerateWebPages(object):
    def __init__(self, getConfigValue, pageDir, resourceNames,
                 pageTitle, pageSubTitles, pageVersion, extraVersions, descriptionInfo):
        self.pageTitle = pageTitle
        self.pageSubTitles = pageSubTitles
        self.pageVersion = pageVersion
        self.extraVersions = extraVersions
        self.pageDir = pageDir
        self.pagesOverview = OrderedDict()
        self.pagesDetails = OrderedDict()
        self.getConfigValue = getConfigValue
        self.resourceNames = resourceNames
        self.descriptionInfo = descriptionInfo
        self.diag = logging.getLogger("GenerateWebPages")

    def makeSelectors(self, subPageNames, tags=[]):
        allSelectors = []
        firstSubPageName = self.getConfigValue("historical_report_subpages", "default")[0]
        for subPageName in subPageNames:
            if subPageName == firstSubPageName:
                suffix = ""
            else:
                suffix = "_" + subPageName.lower()
            allSelectors.append(Selector(subPageName, suffix, self.getConfigValue, tags))
        return allSelectors

    def removeUnused(self, unused, tagData):
        successTags = {}
        for tag in unused:
            for fn in tagData.get(tag):
                if os.path.basename(fn).startswith("teststate_"):
                    os.remove(fn)
                else:
                    successTags.setdefault(fn, []).append(tag)
        for fn, tagsToRemove in list(successTags.items()):
            linesToKeep = []
            with open(fn) as readFile:
                for line in readFile:
                    tag = line.strip().split()[0]
                    if tag not in tagsToRemove:
                        linesToKeep.append(line)

            with open(fn, "w") as writeFile:
                for line in linesToKeep:
                    writeFile.write(line)

    def generate(self, repositoryDirs, subPageNames, archiveUnused):
        minorVersionHeader = HTMLgen.Container()
        allMonthSelectors = set()
        latestMonth = None
        pageToGraphs = {}
        for version, repositoryDirInfo in list(repositoryDirs.items()):
            self.diag.info("Generating " + version)
            tagData, stateFiles, successFiles = self.findTestStateFilesAndTags(repositoryDirInfo)
            if len(stateFiles) > 0 or len(successFiles) > 0:
                tags = list(tagData.keys())
                tags.sort(key=self.tagSortKey)
                selectors = self.makeSelectors(subPageNames, tags)
                monthSelectors = SelectorByMonth.makeInstances(tags)
                allMonthSelectors.update(monthSelectors)
                allSelectors = selectors + list(reversed(monthSelectors))
                # If we already have month pages, we only regenerate the current one
                if len(self.getExistingMonthPages()) == 0:
                    selectors = allSelectors
                else:
                    currLatestMonthSel = monthSelectors[-1]
                    if latestMonth is None or currLatestMonthSel.linkName == latestMonth:
                        selectors.append(monthSelectors[-1])
                        latestMonth = currLatestMonthSel.linkName
                    selectedTags = set()
                    unusedTags = set(tags)
                    for selector in selectors:
                        currTags = set(selector.selectedTags)
                        selectedTags.update(currTags)
                        if archiveUnused:
                            unusedTags.difference_update(currTags)
                    tags = [t for t in tags if t in selectedTags]
                    if archiveUnused and unusedTags:
                        plugins.log.info(
                            "Automatic repository cleaning will now remove old data for the following runs:")
                        for tag in sorted(unusedTags, key=self.tagSortKey):
                            plugins.log.info("- " + tag)
                        plugins.log.info(
                            "(To disable automatic repository cleaning in future, please run with the --manualarchive flag when collating the HTML report.)")
                        self.removeUnused(unusedTags, tagData)

                loggedTests = OrderedDict()
                categoryHandlers = {}
                self.diag.info("Processing " + str(len(stateFiles)) + " teststate files")
                relevantFiles = 0
                for stateFile, repository in stateFiles:
                    tag = self.getTagFromFile(stateFile)
                    if len(tags) == 0 or tag in tags:
                        relevantFiles += 1
                        testId, state, extraVersion = self.processTestStateFile(stateFile, repository)
                        loggedTests.setdefault(extraVersion, OrderedDict()).setdefault(
                            testId, OrderedDict())[tag] = state
                        categoryHandlers.setdefault(tag, CategoryHandler()).registerInCategory(
                            testId, state.category, extraVersion, state)
                        if relevantFiles % 100 == 0:
                            self.diag.info("- Processed " + str(relevantFiles) + " files with matching tags so far")
                self.diag.info("Processed " + str(relevantFiles) + " relevant teststate files")
                self.diag.info("Processing " + str(len(successFiles)) + " success files")
                for successFile, repository in successFiles:
                    testId = self.getTestIdentifier(successFile, repository)
                    extraVersion = self.findExtraVersion(repository)
                    with open(successFile) as f:
                        fileTags = set()
                        for line in f:
                            parts = line.strip().split(" ", 1)
                            if len(parts) != 2:
                                continue
                            tag, text = parts
                            if tag in fileTags:
                                sys.stderr.write("WARNING: more than one result present for tag '" +
                                                 tag + "' in file " + successFile + "!\n")
                                sys.stderr.write("Ignoring later ones\n")
                                continue

                            fileTags.add(tag)
                            if len(tags) == 0 or tag in tags:
                                loggedTests.setdefault(extraVersion, OrderedDict()).setdefault(
                                    testId, OrderedDict())[tag] = text
                                categoryHandlers.setdefault(tag, CategoryHandler()).registerInCategory(
                                    testId, "success", extraVersion, text)
                self.diag.info("Processed " + str(len(successFiles)) + " success files")
                versionToShow = self.removePageVersion(version)
                hasData = False
                for sel in selectors:
                    filePath = self.getPageFilePath(sel)
                    if filePath in self.pagesOverview:
                        page, pageColours = self.pagesOverview[filePath]
                    else:
                        page = self.createPage()
                        pageColours = set()
                        self.pagesOverview[filePath] = page, pageColours

                    tableHeader = self.getTableHeader(version, repositoryDirs)
                    heading = self.getHeading(versionToShow)
                    hasNewData, graphLink, tableColours = self.addTable(page, self.resourceNames, categoryHandlers, version,
                                                                        loggedTests, sel, tableHeader, filePath, heading, repositoryDirInfo)
                    hasData |= hasNewData
                    pageColours.update(tableColours)
                    if graphLink:
                        pageToGraphs.setdefault(page, []).append(graphLink)

                if hasData and versionToShow:
                    link = HTMLgen.Href("#" + version, versionToShow)
                    minorVersionHeader.append(link)

                # put them in reverse order, most relevant first
                linkFromDetailsToOverview = [sel.getLinkInfo(self.pageVersion) for sel in allSelectors]
                for tag in tags:
                    details = self.pagesDetails.setdefault(tag, TestDetails(tag, self.pageTitle, self.pageSubTitles))
                    details.addVersionSection(version, categoryHandlers[tag], linkFromDetailsToOverview)

        selContainer = HTMLgen.Container()
        selectors = self.makeSelectors(subPageNames)
        for sel in selectors:
            target, linkName = sel.getLinkInfo(self.pageVersion)
            selContainer.append(HTMLgen.Href(target, linkName))

        monthContainer = HTMLgen.Container()
        if len(allMonthSelectors) == 1:
            # Don't want just one month, no navigation possible
            prevMonth = list(allMonthSelectors)[0].getPreviousMonthSelector()
            allMonthSelectors.add(prevMonth)

        for sel in sorted(allMonthSelectors, key=lambda s: s.sortKey()):
            target, linkName = sel.getLinkInfo(self.pageVersion)
            monthContainer.append(HTMLgen.Href(target, linkName))

        for page, pageColours in list(self.pagesOverview.values()):
            if len(monthContainer.contents) > 0:
                page.prepend(HTMLgen.Heading(2, monthContainer, align='center'))
            graphs = pageToGraphs.get(page)
            page.prepend(HTMLgen.Heading(2, selContainer, align='center'))
            if minorVersionHeader.contents:
                if not graphs is None and len(graphs) > 1:
                    page.prepend(HTMLgen.Heading(1, *graphs, align='center'))
                page.prepend(HTMLgen.Heading(1, minorVersionHeader, align='center'))
            creationDate = TitleWithDateStamp("").__str__().strip()
            page.prepend(HTMLgen.Paragraph(creationDate, align="center"))
            page.prepend(HTMLgen.Heading(1, self.getHeading(), align='center'))
            if len(pageColours) > 0:
                page.prepend(HTMLgen.BR())
                page.prepend(HTMLgen.BR())
                page.script = self.getFilterScripts(pageColours)

        self.writePages()

    def getFilterScripts(self, pageColours):
        finder = ColourFinder(self.getConfigValue)
        rowHeaderColour = finder.find("row_header_bg")
        successColour = finder.find("success_bg")
        # Always put green at the start, we often want to filter that
        sortedColours = sorted(pageColours, key=lambda c: (c != successColour, c))
        scriptCode = "var TEST_ROW_HEADER_COLOR = " + repr(rowHeaderColour) + ";\n" + \
                     "var Colors = " + repr(sortedColours) + ";"
        return [HTMLgen.Script(code=scriptCode),
                HTMLgen.Script(src="../javascript/jquery.js"),
                HTMLgen.Script(src="../javascript/filter.js"),
                HTMLgen.Script(src="../javascript/comment.js")]

    def getHeading(self, versionToShow=""):
        heading = "Test results for " + self.pageTitle
        if versionToShow:
            heading += "." + versionToShow
        return heading

    def getTableHeader(self, version, repositoryDirs):
        return version if len(repositoryDirs) > 1 else ""

    def getExistingMonthPages(self):
        return glob(os.path.join(self.pageDir, "test_" + self.pageVersion + "_all_???[0-9][0-9][0-9][0-9].html"))

    def tagSortKey(self, x):
        return self.getTagTimeInSeconds(x), plugins.padNumbersWithZeroes(x)

    def getTagFromFile(self, fileName):
        return os.path.basename(fileName).replace("teststate_", "")

    def findTestStateFilesAndTags(self, repositoryDirs):
        tagData, stateFiles, successFiles = {}, [], []
        for _, dir in repositoryDirs:
            self.diag.info("Looking for teststate files in " + dir)
            for root, _, files in sorted(os.walk(dir)):
                for file in files:
                    path = os.path.join(root, file)
                    if file.startswith("teststate_"):
                        tag = self.getTagFromFile(file)
                        stateFiles.append((path, dir))
                        tagData.setdefault(tag, []).append(path)
                    elif file.startswith("succeeded_"):
                        successFiles.append((path, dir))
                        with open(path) as f:
                            for line in f:
                                parts = line.split()
                                if parts:
                                    tag = parts[0]
                                    tagData.setdefault(tag, []).append(path)

            self.diag.info("Found " + str(len(stateFiles)) + " teststate files and " +
                           str(len(successFiles)) + " success files in " + dir)
        return tagData, stateFiles, successFiles

    def processTestStateFile(self, stateFile, repository):
        state = self.readState(stateFile)
        testId = self.getTestIdentifier(stateFile, repository)
        extraVersion = self.findExtraVersion(repository)
        return testId, state, extraVersion

    def findExtraVersion(self, repository):
        versions = os.path.basename(repository).split(".")
        for i in range(len(versions)):
            version = ".".join(versions[i:])
            if version in self.extraVersions:
                return version
        return ""

    @classmethod
    def readState(cls, stateFile):
        file = open(stateFile, "rb")
        try:
            state = plugins.getNewTestStateFromFile(file)
            if isinstance(state, plugins.TestState):
                return state
            else:
                return cls.readErrorState("Incorrect type for state object.")
        except Exception as e:
            if os.path.getsize(stateFile) > 0:
                return cls.readErrorState("Stack info follows:\n" + str(e))
            else:
                return plugins.Unrunnable("Results file was empty, probably the disk it resides on is full.", "Disk full?")

    @staticmethod
    def readErrorState(errMsg):
        freeText = "Failed to read results file, possibly deprecated format. " + errMsg
        return plugins.Unrunnable(freeText, "read error")

    def removePageVersion(self, version):
        leftVersions = []
        pageSubVersions = self.pageVersion.split(".")
        for subVersion in version.split("."):
            if not subVersion in pageSubVersions:
                leftVersions.append(subVersion)
        return ".".join(leftVersions)

    def getPageFilePath(self, selector):
        pageName = selector.getLinkInfo(self.pageVersion)[0]
        return os.path.join(self.pageDir, pageName)

    def createPage(self):
        style = "body,td {color: #000000;font-size: 11px;font-family: Helvetica;} th {color: #000000;font-size: 13px;font-family: Helvetica;}"
        title = "Test results for " + self.pageTitle
        return HTMLgen.SimpleDocument(title=title, style=style, xhtml=True, meta='<meta charset="' + locale.getpreferredencoding() + '">')

    def makeTableHeaderCell(self, tableHeader):
        container = HTMLgen.Container()
        container.append(HTMLgen.Name(tableHeader))
        container.append(HTMLgen.U(HTMLgen.Heading(1, tableHeader, align='center')))
        return HTMLgen.TD(container)

    def makeImageLink(self, graphFile):
        image = HTMLgen.Image(filename=graphFile, src=graphFile, height=100, width=150, border=0)
        return HTMLgen.Href(graphFile, image)

    def getTestTable(self, *args):
        return TestTable(*args)

    def addTable(self, page, resourceNames, categoryHandlers, version, loggedTests, selector, tableHeader, filePath, graphHeading, repositoryDirs):
        graphDirname, graphFileRef = self.getGraphFileParts(filePath, version)
        testTable = self.getTestTable(self.getConfigValue, resourceNames, self.descriptionInfo,
                                      selector.selectedTags, categoryHandlers, self.pageVersion, version, os.path.join(graphDirname, graphFileRef))
        table = testTable.generate(loggedTests, self.pageDir, repositoryDirs)
        if table:
            cells = []
            if tableHeader:
                page.append(HTMLgen.HR())
                cells.append(self.makeTableHeaderCell(tableHeader))

            graphLink = None
            fullPath = os.path.abspath(os.path.join(graphDirname, graphFileRef))
            if testTable.generateGraph(fullPath, graphHeading):
                graphLink = self.makeImageLink(graphFileRef.replace("\\", "/"))
                cells.append(HTMLgen.TD(graphLink))

            if len(cells):
                row = HTMLgen.TR(*cells)
                initialTable = HTMLgen.TableLite(align="center")
                initialTable.append(row)
                page.append(initialTable)

            extraVersions = list(loggedTests.keys())[1:]
            if len(extraVersions) > 0:
                page.append(testTable.generateExtraVersionLinks(extraVersions))

            page.append(table)
            return True, graphLink, testTable.usedColours
        else:
            return False, None, []

    def getGraphFileParts(self, filePath, version):
        dirname, local = os.path.split(filePath)
        versionSuffix = self.removePageVersion(version)
        if versionSuffix:
            versionSuffix = "." + versionSuffix
        return dirname, os.path.join("images", local[:-5] + versionSuffix + ".png")

    def writePages(self):
        plugins.log.info("Writing overview pages...")
        fileToUrl = self.getConfigValue("file_to_url", allSubKeys=True)
        for pageFile, (page, _) in list(self.pagesOverview.items()):
            page.write(pageFile)
            plugins.log.info("wrote: '" + plugins.relpath(pageFile, self.pageDir) + "'")
            if fileToUrl:
                url = convertToUrl(pageFile, fileToUrl)
                plugins.log.info("(URL is " + url + ")")
        plugins.log.info("Writing detail pages...")
        for tag, details in list(self.pagesDetails.items()):
            pageName = getDetailPageName(self.pageVersion, tag)
            details.write(os.path.join(self.pageDir, pageName))
            plugins.log.info("wrote: '" + pageName + "'")
        self.writeCommentListPage()

    def writeCommentListPage(self):
        filename = os.path.join(self.pageDir, "commentlist.html")
        plugins.log.info("Writing comment html page at " + filename + "...")
        doc = HTMLgen.SimpleDocument(title="All Comments", stylesheet="../javascript/commentliststyle.css", xhtml=True, meta='<meta charset="' + locale.getpreferredencoding() + '">')
        for scriptFile in ["jquery.js", "comment.js", "commentlist.js"]:
            doc.append(HTMLgen.Script(src="../javascript/" + scriptFile))
        doc.write(filename)

    def getTestIdentifier(self, stateFile, repository):
        dir = os.path.dirname(stateFile)
        return dir.replace(repository + os.sep, "").replace(os.sep, " ")

    def getTagTimeInSeconds(self, tag):
        timePart = tag.split("_")[0]
        return time.mktime(time.strptime(timePart, "%d%b%Y"))


class TestTable:
    def __init__(self, getConfigValue, resourceNames, descriptionInfo, tags, categoryHandlers, pageVersion, version, graphFilePath):
        self.getConfigValue = getConfigValue
        self.resourceNames = resourceNames
        self.colourFinder = ColourFinder(getConfigValue)
        self.descriptionInfo = descriptionInfo
        self.tags = tags
        self.categoryHandlers = categoryHandlers
        self.pageVersion = pageVersion
        self.version = version
        self.graphFilePath = graphFilePath  # For convenience in performance analyzer.
        self.usedColours = set()

    def generateGraph(self, fileName, heading):
        if len(self.tags) > 1:  # Don't bother with graphs when tests have only run once
            try:
                from .resultgraphs import GraphGenerator
            except Exception as e:
                sys.stderr.write("Not producing result graphs: " + str(e) + "\n")
                return False  # if matplotlib isn't installed or is too old

            data = self.getColourKeySummaryData()
            generator = GraphGenerator()
            generator.generateGraph(fileName, heading, data, self.colourFinder)
            return True
        else:
            return False

    def generate(self, loggedTests, pageDir, repositoryDirs):
        table = HTMLgen.TableLite(border=0, cellpadding=4, cellspacing=2, width="100%")
        table.append(self.generateTableHead(repositoryDirs))
        table.append(self.generateSummaries())
        if os.getenv("JENKINS_URL"):
            changeRow = self.generateJenkinsChanges(pageDir)
            if changeRow:
                table.append(changeRow)
        hasRows = False
        for extraVersion, testInfo in list(loggedTests.items()):
            currRows = []
            for test in sorted(testInfo.keys()):
                results = testInfo[test]
                rows = self.generateTestRows(test, extraVersion, results)
                if rows:
                    currRows += rows

            if len(currRows) == 0:
                continue
            else:
                hasRows = True

            # Add an extra line in the table only if there are several versions.
            if len(loggedTests) > 1:
                fullVersion = self.version
                if extraVersion:
                    fullVersion += "." + extraVersion
                table.append(self.generateExtraVersionHeader(fullVersion))
                table.append(self.generateSummaries(extraVersion))

            for row in currRows:
                table.append(row)

        if hasRows:
            table.append(HTMLgen.BR())
            return table

    def findJenkinsChanges(self, prevTag, tag, cacheDir):
        buildNumber = self.getJenkinsBuildNumber(tag)
        cacheFileOldName = os.path.join(cacheDir, buildNumber)
        cacheFile = os.path.join(cacheDir, tag)
        if os.path.isfile(cacheFileOldName):
            os.rename(cacheFileOldName, cacheFile)
        if os.path.isfile(cacheFile):
            return eval(open(cacheFile).read().strip())
        else:
            bugSystemData = self.getConfigValue("bug_system_location", allSubKeys=True)
            markedArtefacts = self.getConfigValue("batch_jenkins_marked_artefacts")
            fileFinder = self.getConfigValue("batch_jenkins_archive_file_pattern")
            prevBuildNumber = self.getJenkinsBuildNumber(prevTag) if prevTag else None
            if buildNumber.isdigit() and prevBuildNumber is not None:
                try:
                    allChanges = jenkinschanges.getChanges(
                        prevBuildNumber, buildNumber, bugSystemData, markedArtefacts, fileFinder, cacheDir)
                    plugins.ensureDirectoryExists(cacheDir)
                    with open(cacheFile, "w") as f:
                        f.write(pformat(allChanges) + "\n")
                    return allChanges
                except jenkinschanges.JobStillRunningException:
                    pass  # don't write to cache in this case
            return []

    def getJenkinsBuildNumber(self, tag):
        return tag.split(".")[-1]

    def generateJenkinsChanges(self, pageDir):
        cacheDir = os.path.join(os.path.dirname(pageDir), "jenkins_changes")
        bgColour = self.colourFinder.find("changes_header_bg")
        row = [HTMLgen.TD("Changes", bgcolor=bgColour)]
        hasData = False
        prevTag = None
        for tag in self.tags:
            allChanges = self.findJenkinsChanges(prevTag, tag, cacheDir)
            cont = HTMLgen.Container()
            aborted = False
            for i, (authorOrMessage, target, bugs) in enumerate(allChanges):
                if i:
                    cont.append(HTMLgen.BR())
                if target:
                    cont.append(HTMLgen.Href(target, authorOrMessage))
                else:
                    cont.append(HTMLgen.Font(authorOrMessage, color="red"))
                    aborted = "Aborted" in authorOrMessage
                for bugText, bugTarget in bugs:
                    cont.append(HTMLgen.Href(bugTarget, bugText))
                hasData = True
            row.append(HTMLgen.TD(cont, bgcolor=bgColour))
            if not aborted:
                prevTag = tag
        if hasData:
            return HTMLgen.TR(*row)

    def generateSummaries(self, extraVersion=None):
        bgColour = self.colourFinder.find("column_header_bg")
        row = [HTMLgen.TD("Summary", bgcolor=bgColour)]
        for tag in self.tags:
            categoryHandler = self.categoryHandlers[tag]
            detailPageName = getDetailPageName(self.pageVersion, tag)
            summary = categoryHandler.generateHTMLSummary(detailPageName + "#" + self.version, extraVersion)
            row.append(HTMLgen.TD(summary, bgcolor=bgColour))
        return HTMLgen.TR(*row)

    def getColourKeySummaryData(self):
        fullData = []
        for tag in self.tags:
            colourCount = OrderedDict()
            for colourKey in ["success", "knownbug", "slower", "faster", "performance",
                              "larger", "smaller", "memory", "failure", "incomplete"]:
                colourCount[colourKey] = 0
            categoryHandler = self.categoryHandlers[tag]
            basicData = categoryHandler.getSummaryData()[-1]
            for category, count in basicData:
                colourKey = self.getBackgroundColourKey(category)
                colourCount[colourKey] += count
            fullData.append((getDisplayText(tag), colourCount))
        return fullData

    def generateExtraVersionLinks(self, extraVersions):
        cont = HTMLgen.Container()
        for extra in extraVersions:
            fullName = self.version + "." + extra
            cont.append(HTMLgen.Href("#" + fullName, extra))
        return HTMLgen.Heading(2, cont, align='center')

    def generateExtraVersionHeader(self, extraVersion):
        bgColour = self.colourFinder.find("column_header_bg")
        extraVersionElement = HTMLgen.Container(HTMLgen.Name(extraVersion), extraVersion)
        columnHeader = HTMLgen.TH(extraVersionElement, colspan=len(self.tags) + 1, bgcolor=bgColour)
        return HTMLgen.TR(columnHeader)

    def escapeForHtml(self, text):
        text = html.escape(text, True)
        localeEncoding = locale.getdefaultlocale()[1] or "utf-8"
        return str(text.encode("ascii", "xmlcharrefreplace"), localeEncoding)

    def generateTestRows(self, testName, extraVersion, results):
        bgColour = self.colourFinder.find("row_header_bg")
        testId = makeTestId(self.version, extraVersion, testName)
        description = self.descriptionInfo.get(testName, "")
        container = HTMLgen.Container(HTMLgen.Name(testId), testName)
        rows = []
        testRow = [HTMLgen.TD(container, bgcolor=bgColour, title=self.escapeForHtml(description))]

        # Don't add empty rows to the table
        foundData = False
        bgcol = None
        for tag in self.tags:
            cellContent, bgcol, hasData = self.generateTestCell(tag, testName, testId, results)
            testRow.append(HTMLgen.TD(cellContent, bgcolor=bgcol))
            foundData |= hasData

        if foundData:
            # We only filter based on the final column
            self.usedColours.add(bgcol)
            rows.append(HTMLgen.TR(*testRow))
        else:
            return rows

        for resourceName in self.resourceNames:
            foundData = False
            resourceRow = [HTMLgen.TD(HTMLgen.Emphasis("(" + resourceName + ")"), align="right")]
            for tag in self.tags:
                cellContent, bgcol, hasData = self.generateTestCell(tag, testName, testId, results, resourceName)
                resourceRow.append(HTMLgen.TD(HTMLgen.Emphasis(cellContent), bgcolor=bgcol, align="right"))
                foundData |= hasData

            if foundData:
                rows.append(HTMLgen.TR(*resourceRow))
        return rows

    def getCellData(self, state, resourceName):
        if state:
            if resourceName:
                if hasattr(state, "findComparison"):
                    fileComp = state.findComparison(resourceName, includeSuccess=True)[0]
                    if fileComp:
                        return self.getCellDataFromFileComp(fileComp)
            else:
                return self.getCellDataFromState(state)

        return "N/A", True, self.colourFinder.find("test_default_fg"), self.colourFinder.find("no_results_bg")

    def getCellDataFromState(self, state):
        fileComp = state.getMostSevereFileComparison() if hasattr(state, "getMostSevereFileComparison") else None
        category = state.category if hasattr(state, "category") else "success"
        fgcol, bgcol = self.getColours(category, fileComp)
        if not hasattr(state, "category"):
            brief, hosts = parseState(state)
            return "ok " + brief + compactHostRepr(hosts), True, fgcol, bgcol
        success = category == "success"
        if success:
            cellContent = "ok"
            if state.briefText:
                cellContent += " " + state.briefText
        else:
            cellContent = state.getTypeBreakdown()[1]
        cellContent += " " + compactHostRepr(state.executionHosts)
        return cellContent.strip(), success, fgcol, bgcol

    def getCellDataFromFileComp(self, fileComp):
        success = fileComp.hasSucceeded()
        if success:
            category = "success"
        else:
            category = fileComp.getType()
        fgcol, bgcol = self.getColours(category, fileComp)
        text = fileComp.getSummary()
        return text, success, fgcol, bgcol

    def generateTestCell(self, tag, testName, testId, results, resourceName=""):
        state = results.get(tag)
        cellText, success, fgcol, bgcol = self.getCellData(state, resourceName)
        cellContent = HTMLgen.Font(cellText, color=fgcol)
        if success:
            return cellContent, bgcol, cellText != "N/A"
        else:
            linkTarget = getDetailPageName(self.pageVersion, tag) + "#" + testId
            tooltip = "'" + testName + "' failure for " + getDisplayText(tag)
            return HTMLgen.Href(linkTarget, cellContent, title=tooltip, style="color:black"), bgcol, True

    def getBackgroundColourKey(self, category):
        if category == "success":
            return "success"
        elif category == "bug":
            return "knownbug"
        elif category.startswith("faster"):
            return self.getExistingColourKey(["faster", "performance"])
        elif category.startswith("slower"):
            return self.getExistingColourKey(["slower", "performance"])
        elif category.startswith("smaller"):
            return self.getExistingColourKey(["smaller", "memory"])
        elif category.startswith("larger"):
            return self.getExistingColourKey(["larger", "memory"])
        elif category in ["killed", "unrunnable", "cancelled", "abandoned"]:
            return "incomplete"
        else:
            return "failure"

    def getExistingColourKey(self, keys):
        for key in keys:
            if self.colourFinder.find(key + "_bg"):
                return key

    def getForegroundColourKey(self, bgcolKey, fileComp):
        if (bgcolKey == "performance" and self.getPercent(fileComp) >=
            self.getConfigValue("performance_variation_serious_%", "cputime")) or \
            (bgcolKey == "memory" and self.getPercent(fileComp) >=
             self.getConfigValue("performance_variation_serious_%", "memory")):
            return "performance"
        else:
            return "test_default"

    def getColours(self, category, fileComp):
        bgcolKey = self.getBackgroundColourKey(category)
        fgcolKey = self.getForegroundColourKey(bgcolKey, fileComp)
        return self.colourFinder.find(fgcolKey + "_fg"), self.colourFinder.find(bgcolKey + "_bg")

    def getPercent(self, fileComp):
        return fileComp.perfComparison.percentageChange

    def findTagColour(self, tag):
        return self.colourFinder.find("run_" + getWeekDay(tag) + "_fg")

    def getRunNameDirs(self, repositoryDirs):
        runNameDirs = set()
        for _, dir in repositoryDirs:
            runNameDirs.add(os.path.join(os.path.dirname(os.path.dirname(dir)), "run_names"))
        return runNameDirs

    def getRunEnv(self, runEnv, key):
        return runEnv.get(key, os.getenv(key))

    def generateTableHead(self, repositoryDirs):
        head = [HTMLgen.TH("Test")]
        jenkinsUrl = os.getenv("JENKINS_URL")
        runNameDirs = self.getRunNameDirs(repositoryDirs) if jenkinsUrl else []
        for tag in self.tags:
            tagColour = self.findTagColour(tag)
            linkTarget = getDetailPageName(self.pageVersion, tag)
            linkText = HTMLgen.Font(getDisplayText(tag), color=tagColour)
            buildNumber = self.getJenkinsBuildNumber(tag)
            if jenkinsUrl and buildNumber.isdigit():
                runEnv = getEnvironmentFromRunFiles(runNameDirs, tag)
                container = HTMLgen.Container()
                tooltip = jenkinschanges.getTimestamp(buildNumber)
                container.append(HTMLgen.Href(linkTarget, linkText, title=tooltip))
                container.append(HTMLgen.BR())
                jobTarget = os.path.join(self.getRunEnv(runEnv, "JENKINS_URL"), "job",
                                         self.getRunEnv(runEnv, "JOB_NAME"), buildNumber)
                jobText = HTMLgen.Emphasis(HTMLgen.Font("(Jenkins " + buildNumber + ")", size=1))
                container.append(HTMLgen.Href(jobTarget, jobText, title=tooltip))
                head.append(HTMLgen.TH(container))
            else:
                head.append(HTMLgen.TH(HTMLgen.Href(linkTarget, linkText)))
        heading = HTMLgen.TR()
        heading = heading + head
        return heading


class TestDetails:
    def __init__(self, tag, pageTitle, pageSubTitles):
        tagText = getDisplayText(tag)
        pageDetailTitle = "Detailed test results for " + pageTitle + ": " + tagText
        self.document = HTMLgen.SimpleDocument(title=TitleWithDateStamp(pageDetailTitle), meta='<meta charset="' + locale.getpreferredencoding() + '">')
        headerText = tagText + " - detailed test results for " + pageTitle
        self.document.append(HTMLgen.Heading(1, headerText, align='center'))
        for subTitle, command in pageSubTitles:
            self.document.append(HTMLgen.Center(HTMLgen.Emphasis(subTitle)))
            self.document.append(HTMLgen.Center(HTMLgen.Paragraph(command, style='font-family:monospace')))
        self.totalCategoryHandler = CategoryHandler()
        self.versionSections = []

    def addVersionSection(self, version, categoryHandler, linkFromDetailsToOverview):
        self.totalCategoryHandler.update(categoryHandler)
        container = HTMLgen.Container()
        container.append(HTMLgen.HR())
        container.append(self.getSummaryHeading(version, categoryHandler))
        for desc, testInfo in categoryHandler.getTestsWithDescriptions():
            fullDescription = self.getFullDescription(testInfo, version, linkFromDetailsToOverview)
            if fullDescription:
                container.append(HTMLgen.Name(version + desc))
                container.append(HTMLgen.Heading(3, "Detailed information for the tests that " + desc + ":"))
                container.append(fullDescription)
        self.versionSections.append(container)

    def getSummaryHeading(self, version, categoryHandler):
        return HTMLgen.Heading(2, version + ": " + categoryHandler.generateTextSummary())

    def write(self, fileName):
        if len(self.versionSections) > 1:
            self.document.append(self.getSummaryHeading("Total", self.totalCategoryHandler))
        for sect in self.versionSections:
            self.document.append(sect)
        self.versionSections = []  # In case we get called again
        self.document.write(fileName)

    def getFreeTextData(self, tests):
        data = OrderedDict()
        for testName, state, extraVersion in tests:
            freeText = state.freeText if hasattr(state, "freeText") else None
            if freeText:
                if freeText not in data:
                    data[freeText] = []
                data[freeText].append((testName, state, extraVersion))
        return list(data.items())

    def getFullDescription(self, tests, version, linkFromDetailsToOverview):
        freeTextData = self.getFreeTextData(tests)
        if len(freeTextData) == 0:
            return
        fullText = HTMLgen.Container()
        for freeText, tests in freeTextData:
            tests.sort(key=lambda info: info[0])
            for testName, _, extraVersion in tests:
                fullText.append(HTMLgen.Name(makeTestId(version, extraVersion, testName)))
            fullText.append(self.getHeaderLine(tests, version, linkFromDetailsToOverview))
            self.appendFreeText(fullText, freeText)
            if len(tests) > 1:
                for line in self.getTestLines(tests, version, linkFromDetailsToOverview):
                    fullText.append(line)
        return fullText

    def appendFreeText(self, fullText, freeText):
        freeText = freeText.replace("<", "&lt;").replace(">", "&gt;")
        linkMarker = "URL=http"
        if linkMarker in freeText:
            currFreeText = ""
            for line in freeText.splitlines():
                if linkMarker in line:
                    fullText.append(HTMLgen.RawText("<PRE>" + currFreeText.strip() + "</PRE>"))
                    currFreeText = ""
                    words = line.strip().split()
                    linkTarget = words[-1][4:]  # strip off the URL=
                    newLine = " ".join(words[:-1])
                    fullText.append(HTMLgen.Href(linkTarget, newLine))
                    fullText.append(HTMLgen.BR())
                else:
                    currFreeText += line + "\n"
        else:
            currFreeText = freeText
        if currFreeText:
            fullText.append(HTMLgen.RawText("<PRE>" + currFreeText + "</PRE>"))

    def getHeaderLine(self, tests, version, linkFromDetailsToOverview):
        testName, state, extraVersion = tests[0]
        if len(tests) == 1:
            linksToOverview = self.getLinksToOverview(version, testName, extraVersion, linkFromDetailsToOverview)
            headerText = "TEST " + repr(state) + " " + testName + " ("
            container = HTMLgen.Container(headerText, linksToOverview)
            return HTMLgen.Heading(4, container, ")")
        else:
            headerText = str(len(tests)) + " TESTS " + repr(state)
            return HTMLgen.Heading(4, headerText)

    def getTestLines(self, tests, version, linkFromDetailsToOverview):
        lines = []
        for testName, _, extraVersion in tests:
            linksToOverview = self.getLinksToOverview(version, testName, extraVersion, linkFromDetailsToOverview)
            headerText = testName + " ("
            container = HTMLgen.Container(headerText, linksToOverview, ")<br>")
            lines.append(container)
        return lines

    def getLinksToOverview(self, version, testName, extraVersion, linkFromDetailsToOverview):
        links = HTMLgen.Container()
        for targetFile, linkName in linkFromDetailsToOverview:
            testId = makeTestId(version, extraVersion, testName)
            links.append(HTMLgen.Href(targetFile + "#" + testId, linkName))
        return links


class CategoryHandler:
    def __init__(self):
        self.testsInCategory = OrderedDict()

    def update(self, categoryHandler):
        for category, testInfo in list(categoryHandler.testsInCategory.items()):
            testInfoList = self.testsInCategory.setdefault(category, [])
            testInfoList += testInfo

    def registerInCategory(self, testId, category, extraVersion, state):
        self.testsInCategory.setdefault(category, []).append((testId, state, extraVersion))

    def getDescription(self, cat, count):
        shortDescr, _ = getCategoryDescription(cat)
        return str(count) + " " + shortDescr

    def getTestCountDescription(self, count):
        return str(count) + " tests: "

    def generateTextSummary(self):
        numTests, summaryData = self.getSummaryData()
        categoryDescs = [self.getDescription(cat, count) for cat, count in summaryData]
        return self.getTestCountDescription(numTests) + " ".join(categoryDescs)

    def generateHTMLSummary(self, detailPageRef, extraVersion=None):
        numTests, summaryData = self.getSummaryData(extraVersion)
        container = HTMLgen.Container()
        for cat, count in summaryData:
            summary = HTMLgen.Text(self.getDescription(cat, count))
            if cat == "success":
                container.append(summary)
            else:
                linkTarget = detailPageRef + getCategoryDescription(cat)[-1]
                container.append(HTMLgen.Href(linkTarget, summary))

        testCountSummary = HTMLgen.Text(self.getTestCountDescription(numTests))
        return HTMLgen.Container(testCountSummary, container)

    def countTests(self, testInfo, extraVersion):
        if extraVersion is not None:
            return sum((currExtra == extraVersion for (testId, state, currExtra) in testInfo))
        else:
            return len(testInfo)

    def getSummaryData(self, extraVersion=None):
        numTests = 0
        summaryData = []
        for cat, testInfo in list(self.testsInCategory.items()):
            testCount = self.countTests(testInfo, extraVersion)
            if testCount > 0:
                summaryData.append((cat, testCount))
                numTests += testCount
        summaryData.sort(key=self.getSummarySortKey)
        return numTests, summaryData

    def getTestsWithDescriptions(self):
        return sorted([(getCategoryDescription(cat)[1], testInfo) for cat, testInfo in list(self.testsInCategory.items())])

    def getSummarySortKey(self, data):
        # Put success at the start, it's neater like that
        return data[0] != "success", -data[1], data[0]


def getCategoryDescription(cat):
    return plugins.TestState.categoryDescriptions.get(cat, (cat, cat))


def getDetailPageName(pageVersion, tag):
    return "test_" + pageVersion + "_" + tag + ".html"


class BaseSelector(object):
    def __init__(self, linkName, suffix):
        self.selectedTags = []
        self.linkName = linkName
        self.suffix = suffix

    def add(self, tag):
        self.selectedTags.append(tag)

    def getLinkInfo(self, pageVersion):
        return "test_" + pageVersion + self.suffix + ".html", self.linkName

    def sortKey(self):
        pass


class Selector(BaseSelector):
    def __init__(self, linkName, suffix, getConfigValue, tags):
        super(Selector, self).__init__(linkName, suffix)
        cutoff = getConfigValue("historical_report_subpage_cutoff", linkName)
        weekdays = getConfigValue("historical_report_subpage_weekdays", linkName)
        self.selectedTags = tags
        if len(weekdays) > 0:
            self.selectedTags = [tag for tag in self.selectedTags if getWeekDay(tag) in weekdays]
        self.selectedTags = self.selectedTags[-cutoff:]


class SelectorByMonth(BaseSelector):
    @classmethod
    def makeInstances(cls, tags):
        allSelectors = {}
        for tag in tags:
            month = tag[2:9]
            allSelectors.setdefault(month, SelectorByMonth(month)).add(tag)
        return sorted(allSelectors.values(), key=lambda s: s.sortKey())

    def __init__(self, month):
        super(SelectorByMonth, self).__init__(month, "_all_" + month)
        self.month = datetime.strptime(self.linkName, "%b%Y")

    def getPreviousMonthSelector(self):
        first_day_of_current_month = self.month.replace(day=1)
        last_day_of_previous_month = first_day_of_current_month - timedelta(days=1)
        return SelectorByMonth(last_day_of_previous_month.strftime("%b%Y"))

    def sortKey(self):
        return self.month

    def __eq__(self, other):
        return self.linkName == other.linkName

    def __hash__(self):
        return self.linkName.__hash__()
