#!/usr/bin/env python
#
# Copyright 2016 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import codecs
import collections
import re
import string
import sys

from fontTools import ttLib
from fontTools.pens.boundsPen import BoundsPen

from os import path

from nototools import font_data
from nototools import tool_utils
from nototools import unicode_data
from nototools.py23 import basestring
from nototools.py23 import unichr

"""Generate html comparison of codepoints in various fonts."""

_HTML_HEADER_TEMPLATE = """<!DOCTYPE html>
<html lang='en'>
<head>
  <meta charset="utf-8">
  <title>$title</title>
  <style>
    $styles
  </style>
  <style>
    table { background-color: #eee; font-size: 20pt; text-align: center }
    tr.head { font-weight: bold; font-size: 12pt;
        border-style: solid; border-width: 1px; border-color: black;
        border-collapse: separate }
    td:nth-of-type(1), td:nth-last-of-type(3) { font-size: 12pt; text-align:left }
    $mstyles
    td:nth-last-of-type(1), td:nth-last-of-type(2) {
        font-size: 10pt; text-align:left; max-width: 20em }
    .key { background-color: white; font-size: 12pt; border-collapse: separate;
        margin-top: 0; border-spacing: 10px 0; text-align: left }
    .line { font-size: 20pt; word-break: break-all }
    h3 { -webkit-margin-before: 1.75em; -webkit-margin-after: .25em }
    .ctx { font-family: $contextfont }
  </style>
</head>
<body>
  <h3>$title</h3>
"""

_METRICS_STYLES = (
    ", ".join("td:nth-last-of-type(%d)" % i for i in range(4, 9))
    + " { font-size: 10pt; text-align:right; font-family: sansserif }"
)

# hardcoded for now, this assumes 'noto' is one of the defined font names
_CONTEXT_FONT = "noto"

_HTML_FOOTER = """
</body>
</html>
"""


def _cleanlines(textfile):
    """Strip comments and blank lines from textfile, return list of lines."""
    result = []
    with open(textfile, "r") as f:
        for line in f:
            ix = line.find("#")
            if ix >= 0:
                line = line[:ix]
            line = line.strip()
            if line:
                result.append(line)
    return result


class CodeList(object):
    """An ordered list of code points (ints).  These might map to other (PUA) code
    points that the font knows how to display."""

    @staticmethod
    def fromfile(filename):
        if filename.endswith("_codes.txt"):
            return CodeList.frompairfile(filename)
        elif filename.endswith("_cmap.txt"):
            return CodeList.fromrangefile(filename)
        elif filename.endswith(".ttf") or filename.endswith(".otf"):
            return CodeList.fromfontcmap(filename)
        else:
            raise Exception(
                "unrecognized file type %s for CodeList.fromfile" % filename
            )

    @staticmethod
    def fromspec(spec):
        codelist_type, text = [t.strip() for t in spec.split(":")]
        return CodeList.fromtext(text, codelist_type)

    @staticmethod
    def fromtext(text, codelist_type):
        if codelist_type == "cmap":
            return CodeList.fromrangetext(text)
        elif codelist_type == "codes":
            return CodeList.frompairtext(text)
        elif codelist_type == "list":
            return CodeList.fromlisttext(text)
        else:
            raise Exception('unknown codelist type "%s"' % codelist_type)

    @staticmethod
    def fromfontcmap(fontname):
        font = ttLib.TTFont(fontname)
        return CodeList.fromset(font_data.get_cmap(font))

    @staticmethod
    def fromset(cpset):
        return UnicodeCodeList(cpset)

    @staticmethod
    def fromrangetext(cpranges):
        return CodeList.fromset(
            tool_utils.parse_int_ranges(cpranges, allow_compressed=True)
        )

    @staticmethod
    def fromrangefile(cprange_file):
        with open(cprange_file, "r") as f:
            return CodeList.fromrangetext(f.read())

    @staticmethod
    def fromlist(cplist):
        return OrderedCodeList(cplist)

    @staticmethod
    def fromlisttext(cplist):
        codes = tool_utils.parse_int_ranges(
            cplist, allow_duplicates=True, return_set=False, allow_compressed=True
        )
        return CodeList.fromlist(codes)

    @staticmethod
    def fromlistfile(cplist_file):
        return CodeList.fromlisttext(_cleanlines(cplist_file))

    @staticmethod
    def frompairs(cppairs):
        return MappedCodeList(cppairs)

    @staticmethod
    def frompairtext(cppairs_text):
        # if no pairs, will treat as listtext.  cppairs must have only one item
        # or pair per line, however.
        pair_list = None
        single_list = []
        for line in cppairs_text.splitlines():
            parts = [int(s, 16) for s in line.split(";")]
            if pair_list:
                if len(parts) < 2:
                    parts.append(parts[0])
                pair_list.append(tuple(parts)[:2])
            elif len(parts) > 1:
                pair_list = [(cp, cp) for cp in single_list]
                pair_list.append(tuple(parts[:2]))
            else:
                single_list.append(parts[0])

        if pair_list:
            return CodeList.frompairs(pair_list)
        return CodeList.fromlist(single_list)

    @staticmethod
    def frompairfile(cppairs_file):
        return CodeList.frompairtext("\n".join(_cleanlines(cppairs_file)))

    def contains(self, cp):
        """Returns True if cp is in the code list."""
        raise NotImplementedError

    def codes(self):
        """Returns the codes in preferred order."""
        raise NotImplementedError

    def codeset(self):
        """Returns the frozenset of codes."""
        raise NotImplementedError

    def mapped_code(self, cp):
        """Returns the mapped code for this code point."""
        raise NotImplementedError


class UnicodeCodeList(CodeList):
    """A codelist based on unicode code point order with no mapping."""

    def __init__(self, codeset):
        super(CodeList, self).__init__()
        self._codeset = frozenset(codeset)

    def contains(self, cp):
        return cp in self._codeset

    def codes(self):
        return sorted(self._codeset)

    def codeset(self):
        return self._codeset

    def mapped_code(self, cp):
        return cp if cp in self._codeset else None


class MappedCodeList(CodeList):
    def __init__(self, codepairs):
        super(MappedCodeList, self).__init__()
        # hack, TODO: change the column order in the input files
        self._codemap = {v: k for k, v in codepairs}
        self._codes = tuple(p[1] for p in codepairs)

    def contains(self, cp):
        return cp in self._codemap

    def codes(self):
        return self._codes

    def codeset(self):
        return frozenset(self._codes)

    def mapped_code(self, cp):
        return self._codemap.get(cp)


class OrderedCodeList(CodeList):
    def __init__(self, codes):
        super(OrderedCodeList, self).__init__()
        self._codes = tuple(codes)
        self._codeset = frozenset(codes)

    def contains(self, cp):
        return cp in self._codeset

    def codes(self):
        return self._codes

    def codeset(self):
        return self._codeset

    def mapped_code(self, cp):
        return cp if cp in self._codeset else None


def _load_codelist(codelist_spec, data_dir, codelistfile_map):
    for codelist_type in ["file", "cmap", "codes", "list", None]:
        if codelist_type and codelist_spec.startswith(codelist_type + ":"):
            codelist_spec = codelist_spec[len(codelist_type) + 1 :].strip()
            break
    if not codelist_type:
        if codelist_spec.endswith(".txt"):
            codelist_type = "file"
        else:
            raise Exception(
                'cannot determine type of codelist spec "%s"' % codelist_spec
            )
    if codelist_type != "file":
        codelist = CodeList.fromtext(codelist_spec, codelist_type)
    else:
        fullpath = path.join(data_dir, codelist_spec)
        if not path.isfile(fullpath):
            raise Exception('codelist file "%s" not found' % codelist_spec)
        codelist = codelistfile_map.get(fullpath)
        if codelist is None:
            codelist = CodeList.fromfile(fullpath)
            codelistfile_map[codelist_spec] = codelist
    return codelist


class SequenceList(object):
    """A list of strings generated by a spec."""

    def __init__(self, codelists, suffix):
        self.codelists = codelists
        self.suffix = suffix

    def codes(self):
        codes = set()
        for codelist in self.codelists:
            codes |= codelist.codeset()
        codes |= set(ord(cp) for cp in self.suffix)
        return codes

    def __iter__(self):
        for codelist in self.codelists:
            chars = [unichr(cp) for cp in codelist.codes()]
            yield self.suffix.join(chars) + self.suffix


class Target(object):
    """A named collection of data that renders to html or text."""

    @staticmethod
    def from_table_data(name, codelist, used_fonts):
        return CodeTableTarget(name, codelist, used_fonts)

    @staticmethod
    def from_sequence_data(name, codelists, suffix, font):
        sequencelist = SequenceList(codelists, suffix)
        return SequenceListTarget(name, sequencelist, font)

    def __init__(self, name):
        self.name = name

    def name(self):
        return self.name

    def codes(self):
        """Returns the set of codepoints used in this target."""
        raise NotImplementedError

    def generate_text(self, metrics, flag_sets):
        raise NotImplementedError

    def generate_html(self, tindex, context, metrics, flag_sets, cp_to_targets):
        raise NotImplementedError


class SequenceListTarget(Target):
    def __init__(self, name, sequencelist, used_font):
        super(SequenceListTarget, self).__init__(name)
        self.sequencelist = sequencelist
        self.used_font = used_font

    def codes(self):
        return self.sequencelist.codes()

    def generate_text(self, metrics, flag_sets):
        raise NotImplementedError

    def generate_html(self, tindex, context, metrics, flag_sets, cp_to_targets):
        lines = ['<h3 id="target_%d">%s</h3>' % (tindex, self.name)]
        lines.append('<div class="%s line">' % self.used_font[0])
        for seq in self.sequencelist:
            lines.append(seq + "<br/>")
        lines.append("</div>")
        return "\n".join(lines)


class CodeTableTarget(Target):
    def __init__(self, name, codelist, used_fonts):
        super(CodeTableTarget, self).__init__(name)
        self.codelist = codelist
        self.used_fonts = used_fonts

    def codes(self):
        return self.codelist.codes()

    def generate_text(self, metrics, flag_sets):
        lines = [self.name]
        header = ["idx  code"]
        header.extend(f[0] for f in self.used_fonts)
        header.append("age name")
        lines.append(" ".join(header))
        for index, cp in enumerate(self.codelist.codes()):
            line = ["%3d" % index]
            line.append("%5s" % ("%04x" % cp))
            for rkey, keyinfos in self.used_fonts:
                match = any(codelist.contains(cp) for _, _, codelist in keyinfos)
                line.append(rkey if match else ("-" * len(rkey)))
            line.append(unicode_data.age(cp))
            line.append(_flagged_name(cp, flag_sets))
            lines.append(" ".join(line))
        return "\n".join(lines)

    def generate_html(self, tindex, context, metrics, flag_sets, cp_to_targets):
        dump_metrics = False

        if dump_metrics:
            print("$ %s" % self.name)

        def context_string(codelist, cp):
            cps = unichr(codelist.mapped_code(cp))
            return (context % cps) if context else cps

        def _target_line(cp, tindex, tinfo):
            info = []
            for ix, name in tinfo:
                if ix == tindex:
                    continue
                info.append('<a href="#target_%d">%s</a>' % (ix, name))
            if not info:
                return "(no group)"
            return "; ".join(info)

        def _generate_header():
            header_parts = ['<tr class="head"><th>CP']
            for key, _ in self.used_fonts:
                header_parts.append("<th>" + key)
            if metrics is not None:
                header_parts.append("<th>lsb<th>mid<th>rsb<th>wid<th>cy")
            header_parts.append("<th>Age<th>Name")
            return "".join(header_parts)

        if metrics is not None:
            # the metrics apply to the rightmost font
            fontname = self.used_fonts[-1][1][0][0]
            if fontname:
                metrics_font = _get_font(fontname)
            else:
                metrics_font = None
                sys.stderr.write("no metrics font\n")

        lines = ['<h3 id="target_%d">%s</h3>' % (tindex, self.name)]
        char_line = _character_string_html(self.codelist, self.used_fonts[-1])
        if char_line:
            lines.append(char_line)
        lines.append("<table>")
        header = _generate_header()
        linecount = 0
        for cp in self.codelist.codes():
            if linecount % 20 == 0:
                lines.append(header)
            linecount += 1
            line = ["<tr>"]
            line.append("<td>U+%04x" % cp)
            for rkey, keyinfos in self.used_fonts:
                cell_class = None
                cell_text = None
                index = 0
                for font, _, rcodelist in keyinfos:
                    if rcodelist.contains(cp):
                        if len(keyinfos) > 1:
                            cell_class = "%s_%d" % (rkey, index)
                        else:
                            cell_class = rkey
                        cell_class = replace_nonalpha(cell_class)
                        if font:
                            cell_text = context_string(rcodelist, cp)
                        else:
                            cell_text = " * "
                            cell_class += " star"
                        break
                    index += 1
                if cell_class:
                    line.append('<td class="%s">%s' % (cell_class, cell_text))
                else:
                    line.append("<td>&nbsp;")
            name = _flagged_name(cp, flag_sets)
            if metrics is not None:
                cp_metrics = _get_cp_metrics(metrics_font, cp) if metrics_font else None
                if cp_metrics:
                    lsb, rsb, wid, adv, cy = cp_metrics
                    if dump_metrics:
                        print("%04x # %4d, %4d, %4d, %s" % (cp, lsb, adv, cy, name))

                    if cp in metrics:
                        nlsb, nadv, ncy = metrics[cp]
                    else:
                        nlsb, nadv, ncy = lsb, adv, cy
                    nrsb = nadv - wid - nlsb

                    line.append(
                        "<td>%d%s"
                        % (lsb, "&rarr;<b>%d</b>" % nlsb if lsb != nlsb else "")
                    )
                    line.append("<td>%d" % wid)
                    line.append(
                        "<td>%d%s"
                        % (rsb, "&rarr;<b>%d</b>" % nrsb if rsb != nrsb else "")
                    )
                    line.append(
                        "<td>%d%s"
                        % (adv, "&rarr;<b>%d</b>" % nadv if adv != nadv else "")
                    )
                    line.append(
                        "<td>%d%s" % (cy, "&rarr;<b>%d</b>" % ncy if cy != ncy else "")
                    )
                else:
                    line.append("<td><td><td><td><td>")
            line.append("<td>%s" % unicode_data.age(cp))
            line.append("<td>%s" % name)
            line.append("<td>%s" % _target_line(cp, tindex, cp_to_targets.get(cp)))
            lines.append("".join(line))
        lines.append("</table>")
        return "\n".join(lines)


def _load_fonts(data_list, data_dir, codelist_map):
    """data_list is a list of tuples of two to four items.  The first item is
    the key, the second is the name of the font file in data_dir.  The
    second can be None, otherwise it must exist.  The third item, if
    present, is the name to use for the font, otherwise it will be read
    from the font, it must be present where there is no font.  The
    fourth item, if present, is the name of a codelist file, it must be present
    where there is no font.  If present and None, the the unicode cmap from the
    font is used.  otherwise the font file name is stripped of its extension and
    try to find a file from which to create a codelist.
    Multiple tuples can share the same key, these form one column and the order
    of the files composing the tuple defines the order in which they are searched
    for a glyph.
    Returns a list of tuples of key, keyinfo, where keyinfo is
    a list of tuples of filepath, name, codelist."""

    def _load_font(data, codelist_map):
        if len(data) < 4:
            data = data + tuple([None] * (4 - len(data)))
        key, fname, name, codelistfile = data

        if not fname:
            if not name:
                raise Exception("must have name if no font provided")
            if not codelistfile:
                raise Exception("must have codelist file if no font provided")
            fontpath = None
        else:
            fontpath = path.join(data_dir, fname)
            if not path.isfile(fontpath):
                raise Exception('font "%s" not found' % fontpath)

        if codelistfile:
            codelist = _load_codelist(codelistfile, data_dir, codelist_map)

        if fname and (not codelistfile or not name):
            font = ttLib.TTFont(fontpath)
            if not name:
                names = font_data.get_name_records(font)
                name = names[16] if 16 in names else names[1] if 1 in names else None
                if not name:
                    raise Exception('cannot read name from font "%s"' % fontpath)
            if not codelistfile:
                codelist = CodeList.fromset(font_data.get_cmap(font))

        return key, fontpath, name, codelist

    # group by key
    keyorder = []
    keyinfo = collections.defaultdict(list)
    for data in data_list:
        key, fontpath, name, codelist = _load_font(data, codelist_map)
        if key not in keyinfo:
            keyorder.append(key)
        keyinfo[key].append((fontpath, name, codelist))

    return [(key, keyinfo[key]) for key in keyorder]


def _select_used_fonts(codelist, fonts, prefer_fonts, omit_fonts):
    """Return the fonts we want to use to display the codelist, in order.
    If not None, prefer_fonts is a key or list of keys for fonts to order
    at the end.  If not None, omit_fonts is key or list of keys to omit
    even if they would otherwise be used by default, however prefer_fonts
    takes precedence over omit_fonts if the same key is in both."""

    if prefer_fonts is not None:
        if isinstance(prefer_fonts, basestring):
            prefer_fonts = [prefer_fonts]
        preferred = [None] * len(prefer_fonts)
    else:
        prefer_fonts = []
        preferred = []

    if omit_fonts is not None:
        if "_all_" in omit_fonts:
            omit_fonts = [k for k, _ in fonts]
        else:
            omit_fonts = [omit_fonts]
        if prefer_fonts:
            omit_fonts = [k for k in omit_fonts if k not in prefer_fonts]
    else:
        omit_fonts = []

    regular = []
    codes = codelist.codes()
    for f in fonts:
        key, keyinfo = f
        if key in omit_fonts:
            continue
        for name, _, cl in keyinfo:
            if any(cl.contains(cp) for cp in codes):
                is_preferred = False
                for i, k in enumerate(prefer_fonts):
                    if key == k:
                        preferred[i] = f
                        is_preferred = True
                        break
                if not is_preferred:
                    regular.append(f)
                break
    return tuple(regular + filter(None, preferred))


def _load_targets(target_data, fonts, data_dir, codelist_map):
    """Target data is a list of tuples of target names, codelist files, an
    optional preferred font key or list of keys, and an optional omitted font
    key or list of keys. All files should be in data_dir.  Codelist_map is a
    cache in case the codelist file has already been read.  Returns a list of
    tuples of target name, codelist, and fontlist."""

    def _create_suffix(charlist):
        return charlist.decode("unicode-escape")

    def _select_font(fonts, font_id):
        for f in fonts:
            if f[0] == font_id:
                return f
        raise Exception('no font with id "%s"' % font_id)

    result = []
    for target in target_data:
        target_type, name, codelist_spec = target[:3]
        if target_type == "table":
            codelist = _load_codelist(codelist_spec, data_dir, codelist_map)
            prefer_fonts = target[3] if len(target) > 3 else None
            omit_fonts = target[4] if len(target) > 4 else None
            used_fonts = _select_used_fonts(codelist, fonts, prefer_fonts, omit_fonts)
            if not used_fonts:
                raise Exception("no fonts used by target %s" % name)
            result.append(Target.from_table_data(name, codelist, used_fonts))
        elif target_type == "sequence":
            if len(target) < 5:
                raise Exception("sequence target too short")
            lists = codelist_spec.split(",")
            codelists = [CodeList.fromlisttext(cl) for cl in lists]
            suffix = _create_suffix(target[3])
            font_tuple = _select_font(fonts, target[4])
            result.append(
                Target.from_sequence_data(name, codelists, suffix, font_tuple)
            )
    return tuple(result)


def _create_codeset_from_expr(expr_list, flag_sets, data_dir, codelist_map):
    """Processes expr_list in order, building a codeset.
    See _read_flag_data_from_file for information on expr_list.
    This can modify flag_sets and codelist_map."""

    result = ()
    for op, exp in expr_list:
        if exp not in flag_sets:
            # its a codelist
            codes = _load_codelist(exp, data_dir, codelist_map).codeset()
        else:
            codes_or_spec = flag_sets[exp]
            if isinstance(codes_or_spec, (set, frozenset)):
                codes = codes_or_spec
            else:
                # replace the spec with the actual codes
                if codes_or_spec is None:
                    # we only know about '_emoji_' and '_math_'
                    if exp == "_emoji_":
                        codes = (
                            unicode_data.get_emoji()
                            - unicode_data.get_unicode_emoji_variants("proposed_extra")
                        )
                    elif exp == "_math_":
                        codes = unicode_data.chars_with_property("Math")
                    else:
                        raise Exception('unknown special codeset "%s"' % exp)
                else:
                    codes = _load_codelist(
                        codes_or_spec, data_dir, codelist_map
                    ).codeset()
                flag_sets[exp] = codes
        if op == "|":
            if not result:
                # it appers that python 'optimizes' |= by replacing the lhs by rhs if
                # lhs is an empty set, but this changes the type of lhs to frozenset...
                result = set(codes)
            else:
                result |= codes
        elif op == "&":
            result &= codes
        elif op == "-":
            result -= codes
        else:
            raise Exception('unknown op "%s"' % op)

    return result


def _load_flags(flag_data, data_dir, codelist_map):
    """Flag data is a list of tuples of defined sets or flags and expressions, see
    _read_flag_data_from_file for more info.
    This returns a map from set name to a tuple of (cp_set, bool) where True
    means the flag is set for a cp if it is in the cp_set, and false means the
    flag is set if the cp is not in the cp_set.

    This can fail since the code processing the flag_data does not actually try
    to load the codelists."""

    flag_sets = {}
    flag_map = {}
    for flag_info in flag_data:
        t0, t1, t2 = flag_info
        if t0 == "!define":
            set_name = t1
            if set_name in ["_emoji_", "_math_"]:
                set_codes = None  # gets created by _create_codeset_from_expr
            else:
                set_codes = _load_codelist(t2, data_dir, codelist_map).codeset()
            flag_sets[set_name] = set_codes
        else:
            flag_name = t0
            flag_in = t1
            flag_set = _create_codeset_from_expr(t2, flag_sets, data_dir, codelist_map)
            flag_map[flag_name] = (flag_set, flag_in)
    return flag_map


def _load_fonts_targets_flags(font_data, target_data, flag_data, data_dir):
    # we cache the codelists to avoid building them twice if they're referenced by
    # both fonts and targets, not a big deal but...
    codelist_map = {}
    fonts = _load_fonts(font_data, data_dir, codelist_map)
    targets = _load_targets(target_data, fonts, data_dir, codelist_map)
    flags = _load_flags(flag_data, data_dir, codelist_map)
    return fonts, targets, flags


def strip_comments_from_file(filename):
    with open(filename, "r") as f:
        for line in f:
            ix = line.find("#")
            if ix >= 0:
                line = line[:ix]
            line = line.strip()
            if not line:
                continue
            yield line


def _read_font_data_from_file(filename):
    font_data = []
    for line in strip_comments_from_file(filename):
        info = line.split(";")
        while len(info) < 4:
            info.append(None)
        font_data.append(tuple(info))
    return font_data


def _read_target_data_from_file(filename):
    """Target data uses # to indicate a comment to end of line.
    Comments are stripped, then an empty or blank line is ignored.

    Targets are either tables or sequences, the default
    is a table.

    Each line in a table target defines a tuple of four values:
    target name, codelist, preferred font ids, and omitted font
    ids.  Each line in a sequence target defines a tuple of
    four values: target name, codelist, suffix, and font id.
    A line can also start with one of tree directives,
    !define, !default, or !type.

    If a line starts with '!define ' we expect a key followed
    by '=' and then one or more names separated by space. The
    names are turned into a list, and entered into a dictionary
    for the key.  Once defined a key cannot be redefined.

    If a line starts with '!default ' we expect a key of either
    'prefer' or 'omit' optionally followed by '=' and a list of
    names to prefer or omit; these will become the default
    values until the next '!default ' directive.  If there is
    no '=' the value is reset.  An omitted or empty prefer or
    omit field will get the fallback, to explicitly request None
    and override the fallback the field should contain 'None'.

    If a line starts with '!type ' we expect either 'table' or
    'sequence' to follow.  This will become the type of the
    following lines until the next '!type ' directive.

    Normally, a line consists of 2-4 fields separated by ';'.
    The first two are a target name and a codelist spec.

    For table targets, the third is the preferred font ids
    separated by space, previously !defined keys can be used
    here instead of this list and the list defined for that key
    will be used.  The fourth is the omitted font ids separated
    by space, they are treated similarly.  If the preferred or
    omit field is missing or empty and a default value for it
    has been set, that value is used.

    For sequence targets, the third is a hex sequence indicating
    the suffix string to apply after each codepoint, and the
    fourth is the font id; these must both be present.

    This returns a list of the tuples of the type name followed
    by the data for that type.
    """

    def add_index_list_or_defined(info, index, fallback, defines):
        """Extend or update info[index], possibly using defines"""
        if len(info) <= index:
            info.append(fallback)
        elif info[index] is not None:
            item = info[index]
            if item in defines:
                items = defines[item]
            elif item == "None":
                items = None
            elif item:
                items = item.split()
            else:
                items = fallback
            info[index] = items

    prefer_fallback = None
    omit_fallback = None
    target_type = "table"
    defines = {}
    target_data = []
    kDefineDirective = "!define "
    kDefaultDirective = "!default "
    kTypeDirective = "!type "

    for line in strip_comments_from_file(filename):
        if line.startswith(kDefineDirective):
            # !define key=val val...
            name, rest = line[len(kDefineDirective) :].split("=")
            name = name.strip()
            if name in defines:
                raise Exception("name %s already defined in %s" % (name, filename))
            rest = rest.strip().split()
            defines[name] = tuple(rest)
            continue
        if line.startswith(kDefaultDirective):
            # !default prefer|omit=val val...
            values = line[len(kDefaultDirective) :].split("=")
            name = values[0].strip()
            rest = values[1].strip().split() if len(values) > 1 else None
            if not rest:
                rest = None
            if name == "prefer":
                prefer_fallback = rest
            elif name == "omit":
                omit_fallback = rest
            else:
                raise Exception("default only understands 'prefer' or 'omit'")
            continue
        if line.startswith(kTypeDirective):
            # !type table|sequence
            value = line[len(kTypeDirective) :]
            if value in {"table", "sequence"}:
                target_type = value
            else:
                raise Exception("type only understands 'table' or 'sequence'")
            continue
        info = [k.strip() for k in line.split(";")]
        if len(info) < 2:
            raise Exception('need at least two fields in "%s"' % line)
        if target_type == "table":
            # name;character spec or filename;prefer_id... or empty;omit_id... or empty
            add_index_list_or_defined(info, 2, prefer_fallback, defines)  # preferred
            add_index_list_or_defined(info, 3, omit_fallback, defines)  # omitted
            target_data.append(tuple(["table"] + info))
        elif target_type == "sequence":
            if len(info) < 4:
                raise Exception('need four fields in sequence data in "%s"' % line)
            target_data.append(tuple(["sequence"] + info))

    return target_data


def _flagged_name(cp, flag_sets):
    """Prepend any flags to cp's unicode name, and return.  Flag_sets
    is a map from flag name to a tuple of cp set and boolean.
    True means add flag if cp in set, False means add flag if it is
    not in the set."""
    try:
        name = unicode_data.name(cp)
    except:
        raise Exception("no name for %04X" % cp)
    flags = []
    for k, v in sorted(flag_sets.items()):
        if (cp in v[0]) == v[1]:
            flags.append(k)
    if flags:
        name = "(%s) %s" % (", ".join(flags), name)
    return name


def generate_text(outfile, title, fonts, targets, flag_sets, metrics, data_dir):
    outfile.write(title + "\n")
    outfile.write("\n")
    outfile.write("Fonts:\n")
    max_keylen = max(len(key) for key, _ in fonts)
    fmt = "  %%%ds: %%s (%%s)" % max_keylen
    for key, keyinfos in fonts:
        for font, name, _ in keyinfos:
            rel_font = path.relpath(font, data_dir) if font else "(no font)"
            outfile.write(fmt % (key, name, rel_font) + "\n")
    outfile.write("\n")

    for target in targets:
        outfile.write("\n")
        outfile.write(target.generate_text(flag_sets, metrics) + "\n")


def _generate_fontkey(fonts, targets, data_dir):
    lines = ['<p style="margin-bottom:5px"><b>Targets</b>']
    lines.append('<div style="margin-left:20px"><table class="key">')
    for tid, target in enumerate(targets):
        lines.append('<tr><th><a href="#target_%s">%s</a>' % (tid, target.name))
    lines.append("</table></div>")

    lines.append('<p style="margin-bottom:5px"><b>Fonts</b>')
    lines.append('<div style="margin-left:20px"><table class="key">')
    for key, keyinfos in fonts:
        for font, name, _ in keyinfos:
            rel_font = path.relpath(font, data_dir) if font else "(no font)"
            lines.append("<tr><th>%s<td>%s<td>%s" % (key, name, rel_font))
    lines.append("</table></div>")

    return "\n".join(lines)


_nonalpha_re = re.compile(r"\W")


def replace_nonalpha(key):
    return _nonalpha_re.sub("_", key)


def _generate_styles(fonts, relpath):
    face_pat = """@font-face {
      font-family: "%s"; src:url("%s")
    }"""

    facelines = []
    classlines = []
    for key, keyinfos in fonts:
        index = 0
        for font, _, _ in keyinfos:
            if len(keyinfos) > 1:
                kname = "%s_%d" % (replace_nonalpha(key), index)
            else:
                kname = replace_nonalpha(key)
            index += 1
            if not font:
                classlines.append(".%s { font-size: 12pt }" % kname)
            else:
                if relpath is None:
                    font = "file://" + font
                else:
                    font = path.join(relpath, path.basename(font))
                facelines.append(face_pat % (kname, font))
                classlines.append(
                    '.%s { font-family: "%s", "noto_0" }' % (kname, kname)
                )

    lines = []
    lines.extend(facelines)
    lines.append("")
    lines.extend(classlines)
    return "\n    ".join(lines)


def _character_string_html(codelist, used_font):
    C0_controls = frozenset(range(0, 0x20))
    rkey, rinfo = used_font
    _, _, f_codelist = rinfo[0]
    f_codeset = frozenset(f_codelist.codeset() - C0_controls)
    cps = [cp for cp in codelist.codes() if cp in f_codeset]
    if not cps:
        return None
    line = ['<bdo class="', rkey, ' line" dir="ltr">']
    line.extend(unichr(cp) for cp in cps)
    line.append("</bdo>")
    return "".join(line)


_FONT_CACHE = {}


def _get_font(fontname):
    font = _FONT_CACHE.get(fontname)
    if not font:
        font = ttLib.TTFont(fontname)
        _FONT_CACHE[fontname] = font
    return font


GMetrics = collections.namedtuple("GMetrics", "lsb, rsb, wid, adv, cy")


def _get_cp_metrics(font, cp):
    # returns metrics for nominal glyph for cp, or None if cp not in font
    cmap = font_data.get_cmap(font)
    if cp not in cmap:
        return None
    glyphs = font.getGlyphSet()
    g = glyphs[cmap[cp]]
    pen = BoundsPen(glyphs)
    g.draw(pen)
    if not pen.bounds:
        return None
    xmin, ymin, xmax, ymax = pen.bounds
    return GMetrics(xmin, g.width - xmax, xmax - xmin, g.width, (ymin + ymax) / 2)


_expr_re = re.compile(r"(\||&|(?<![0-9a-fA-F])-(?![0-9a-fA-F]))")


def _scan_expr(expr, def_names, used_names):
    """Scans the expression, building a list of operation tuples."""
    result = []
    op_str = "|"
    while expr:
        op = op_str
        m = _expr_re.search(expr)
        if not m:
            exp = expr.strip()
            expr = None
            op_str = None
        else:
            exp = expr[: m.start()].strip()
            expr = expr[m.end() :]
            op_str = m.group(1)
        if not exp:
            raise Exception("empty expression after op %s" % op)
        result.append((op, exp))
        if exp in def_names:
            used_names.add(exp)
    return result


def _read_flag_data_from_file(filename):
    """Read flag data file and generate a list of tuples for creating
    the flag data map.  If filename is None, returns an empty list.

    Lines in the file either define a set used by a flag, or define
    a flag.  Define lines start with '!define ' followed by the name
    of the set (_0-9A-Za-z), '=', and the definition (a codelist).

    Definition lines have three fields separated by semicolon,
    the name of the flag, 'in' or 'not in', and the definition
    which can either be a codelist or an expression formed from
    names of !defined sets joined with '&' (intersection), '|'
    (union), or '-' (set difference).  These operations are performed
    in order left to right, there's no predecence.

    Predefined sets are '_emoji_', the unicode extended emoji values,
    and '_math_', codepoints with the 'Math' property.

    '#' is a comment to end-of line.  Blank lines are ignored.

    It's an error if there are multiple defined sets
    with the same name or multiple flags with the same name.

    This returns a list of 3-tuples, one for each set used by a
    flag, then one for each flag.  Tuple for defined sets are
      ('!define', set_name, set_spec),
    there set_spec is None if the set_name is special, like '_emoji_'.
    Tuples for flags are
      (flag_name, True/False, [(op,expr)]),
    where the list of op, expr tuples has the op character
    ('|' '&', '-') and a define name or a codelist."""

    if not filename:
        return []

    predefined = ["_emoji_", "_math_"]

    def_names = set(predefined)

    def_re = re.compile(r"!define ([a-zA-Z][a-zA-Z0-9_]*)\s*=\s*(.*)\s*")
    flag_re = re.compile(r"([^;]+);\s*(in|not in)\s*;\s*(.*)\s*")

    def_info = [("!define", item, None) for item in predefined]
    flag_info = []
    with open(filename, "r") as f:
        for line in f.readlines():
            ix = line.find("#")
            if ix > -1:
                line = line[:ix]
            line = line.strip()
            if not line:
                continue
            if line.startswith("!"):
                m = def_re.match(line)
                if not m:
                    raise Exception('could not match definition line "%s"' % line)
                def_name = m.group(1)
                def_codelist = m.group(2)
                if def_name in def_names:
                    raise Exception(
                        'more than one flag definition named "%s"' % def_name
                    )
                def_names.add(def_name)
                def_info.append(("!define", def_name, def_codelist))
            else:
                m = flag_re.match(line)
                if not m:
                    raise Exception('could not match set definition line "%s"' % line)
                flag_name = m.group(1)
                flag_in_str = m.group(2)
                if flag_in_str == "in":
                    flag_in = True
                elif flag_in_str == "not in":
                    flag_in = False
                else:
                    raise Exception(
                        "found \"%s\" but expected 'in' or 'not in'" % flag_in_str
                    )
                flag_expr = m.group(3)
                flag_info.append([flag_name, flag_in, flag_expr])

    used_names = set()
    flag_expr_info = []
    for flag_name, flag_in, flag_expr in flag_info:
        expr_list = _scan_expr(flag_expr, def_names, used_names)
        flag_expr_info.append((flag_name, flag_in, expr_list))
    used_defs = [t for t in def_info if t[1] in used_names]
    return used_defs + flag_expr_info


"""
def _generate_html_lines(outfile, fontkey):
  ascii_chars = u'#*0123456789 '
  epact_chars = u''.join(unichr(cp) for cp in range(0x102e1, 0x102fb + 1)) + ' '
  phaistos_chars = u''.join(unichr(cp) for cp in range(0x101d0, 0x101fc + 1)) + ' '
  stringlist = [
     ascii_chars,
     u''.join(u'%s\u20e3' % c for c in ascii_chars),
     epact_chars,
     u''.join(u'%s\U000102e0' % c for c in epact_chars),
     phaistos_chars,
     u''.join(u'%s\U000101fd' % c for c in phaistos_chars),
  ]

  lines = ['<h3>Sequences</h3>']
  lines.append('<div class="%s line">' % fontkey)
  for string in stringlist:
    lines.append(string + '<br/>')
  lines.append('</div>')

  outfile.write('\n'.join(lines) + '\n')
"""


def generate_html(
    outfile,
    title,
    fonts,
    targets,
    flag_sets,
    context,
    metrics,
    cp_to_targets,
    data_dir,
    relpath,
):
    """If not None, relpath is the relative path from the outfile to
    the datadir, for use when generating font paths."""

    template = string.Template(_HTML_HEADER_TEMPLATE)
    styles = _generate_styles(fonts, relpath)
    mstyles = _METRICS_STYLES if metrics is not None else ""
    contextfont = _CONTEXT_FONT if context else "sansserif"
    outfile.write(
        template.substitute(
            title=title, styles=styles, mstyles=mstyles, contextfont=contextfont
        )
        + "\n"
    )

    outfile.write(_generate_fontkey(fonts, targets, data_dir) + "\n")

    # hardcode font key for now
    # _generate_html_lines(outfile, 'sym4')

    for index, target in enumerate(targets):
        outfile.write(
            target.generate_html(index, context, metrics, flag_sets, cp_to_targets)
            + "\n"
        )

    outfile.write(_HTML_FOOTER + "\n")


def _build_cp_to_targets(targets):
    """Return a map from cp to a list of pairs of target group index and
    name."""
    cp_to_targets = collections.defaultdict(list)
    #  for i, (name, codelist, _) in enumerate(targets):
    for i, target in enumerate(targets):
        tinfo = (i, target.name)
        for cp in target.codes():
            cp_to_targets[cp].append(tinfo)
    return cp_to_targets


def generate(
    outfile,
    fmt,
    data_dir,
    font_spec,
    target_spec,
    flag_spec,
    title=None,
    context=None,
    metrics=False,
    relpath=None,
):
    if not path.isdir(data_dir):
        raise Exception('data dir "%s" does not exist' % data_dir)

    font_data = _read_font_data_from_file(path.join(data_dir, font_spec))
    target_data = _read_target_data_from_file(path.join(data_dir, target_spec))
    flag_data = _read_flag_data_from_file(
        None if not flag_spec else path.join(data_dir, flag_spec)
    )
    fonts, targets, flag_sets = _load_fonts_targets_flags(
        font_data, target_data, flag_data, data_dir
    )

    if fmt == "txt":
        generate_text(outfile, title, fonts, targets, flag_sets, metrics, data_dir)
    elif fmt == "html":
        cp_to_targets = _build_cp_to_targets(targets)
        generate_html(
            outfile,
            title,
            fonts,
            targets,
            flag_sets,
            context,
            metrics,
            cp_to_targets,
            data_dir,
            relpath,
        )
    else:
        raise Exception('unrecognized format "%s"' % fmt)


def _parse_metrics_file(filename):
    """format is 'cp;lsb;adv' with cp in hex."""
    metrics = {}
    with open(filename, "r") as f:
        for line in f:
            ix = line.find("#")
            if ix >= 0:
                line = line[:ix]
            line = line.strip()
            if not line:
                continue
            cp, lsb, adv, cy = line.split(";")
            cp = int(cp, 16)
            lsb = int(lsb)
            adv = int(adv)
            cy = int(cy)
            if cp in metrics:
                raise Exception("cp %04x listed twice in %s" % (cp, filename))
            metrics[cp] = (lsb, adv, cy)
    return metrics


def _call_generate(
    outfile,
    fmt,
    data_dir,
    font_spec,
    target_spec,
    flag_spec,
    title=None,
    context=None,
    metrics=None,
):
    data_dir = path.realpath(path.abspath(data_dir))
    if metrics is not None:
        if metrics == "-":
            metrics = {}
        else:
            metrics = _parse_metrics_file(path.join(data_dir, metrics))
    if outfile:
        outfile = path.realpath(path.abspath(outfile))
        base, ext = path.splitext(outfile)
        if ext:
            ext = ext[1:]
        if not ext:
            if not fmt:
                fmt = "txt"
                ext = "txt"
            else:
                ext = fmt
        elif not fmt:
            if ext not in ["html", "txt"]:
                raise Exception('don\'t understand "%s" format' % ext)
            fmt = ext
        elif ext != fmt:
            raise Exception(
                'mismatching format "%s" and output extension "%s"' % (fmt, ext)
            )
        outfile = base + "." + ext
        outdir = path.dirname(outfile)
        if data_dir == outdir:
            relpath = ""
        elif data_dir.startswith(outdir):
            relpath = data_dir[len(outdir) + 1 :]
        else:
            relpath = None
        with codecs.open(outfile, "w", "utf-8") as f:
            generate(
                f,
                fmt,
                data_dir,
                font_spec,
                target_spec,
                flag_spec,
                title,
                context,
                metrics,
                relpath,
            )
    else:
        if not fmt:
            fmt = "txt"
        generate(
            sys.stdout,
            fmt,
            data_dir,
            font_spec,
            target_spec,
            flag_spec,
            title,
            context,
            metrics,
        )


def main():
    DEFAULT_OUT = "dingbats_compare"

    parser = argparse.ArgumentParser()
    parser.add_argument(
        "-o",
        "--outfile",
        help="Path to output file (will use %s)" % DEFAULT_OUT,
        const=DEFAULT_OUT,
        metavar="file",
        nargs="?",
    )
    parser.add_argument(
        "-t",
        "--output_type",
        help="output format (defaults based on outfile " 'extension, else "txt")',
        choices=["txt", "html"],
    )
    parser.add_argument(
        "-d",
        "--data_dir",
        help="Path to directory containing fonts " "and data",
        metavar="dir",
        required=True,
    )
    parser.add_argument(
        "--font_spec",
        help="Name of font spec file relative to data dir " "(default 'font_data.txt')",
        metavar="file",
        default="font_data.txt",
    )
    parser.add_argument(
        "--target_spec",
        help="Name of target spec file relative to data dir "
        "(default 'target_data.txt')",
        metavar="file",
        default="target_data.txt",
    )
    parser.add_argument(
        "--flag_spec",
        help="Name of flag spec file relative to data dir "
        "(uses 'flag_data.txt' with no arg)",
        metavar="file",
        nargs="?",
        const="flag_data.txt",
    )
    parser.add_argument(
        "--title",
        help="Title on html page",
        metavar="title",
        default="Character and Font Comparison",
    )
    parser.add_argument(
        "--context",
        help="Context pattern for glyphs (e.g. 'O%%sg')",
        metavar="ctx",
        nargs="?",
        const='<span class="ctx">O</span>%s<span class="ctx">g</span>',
    )
    parser.add_argument(
        "-m",
        "--metrics",
        help="Report metrics of target font, optionally " "with preferred metrics file",
        metavar="file",
        nargs="?",
        const="-",
    )
    args = parser.parse_args()

    _call_generate(
        args.outfile,
        args.output_type,
        args.data_dir,
        args.font_spec,
        args.target_spec,
        args.flag_spec,
        args.title,
        args.context,
        args.metrics,
    )


if __name__ == "__main__":
    main()
