#!/usr/bin/env /usr/bin/python

# Copyright (C) 2015 Ian Harry, Thomas Cokelear
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 3 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.

"""
Tools to create a HTML document that summarizes results produced by pycbc. 
"""
__author__ = "Ian Harry <ian.harry@ligo.org>"
__Id__ = "$Id$"
__name__ = "pycbc_write_results_page"

import sys
import copy
import os
import re
import glob
import shutil
import StringIO
from optparse import *
from glue import markup
from glue import segments
from glue.ligolw import ligolw
from glue.ligolw import lsctables
from glue.ligolw import table
from glue.ligolw import utils
from glue.ligolw.utils import print_tables
from glue.ligolw.utils import segments as segment_utils
from glue.markup import oneliner as e
import glue.pipeline
from pycbc.results import get_library_version_info, get_code_version_numbers

class ContentHandler(ligolw.LIGOLWContentHandler):
    pass
lsctables.use_in(ContentHandler)


# ***************************************************************************
def initialize_page(title,style,script):
  """
  A function that returns a markup.py page object with the required html
  header.
  """
  page = markup.page(mode="strict_html")
  page._escape = False
  doctype="""<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">"""
  doctype+="""\n<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">"""
  page.init(title=title, css=style, script=script , doctype=doctype)
  return page

# ***************************************************************************
def add_page_footer(page):
  """
  A function to add a valid xhtml footer to the bottom of the html.
  
  @param page: The markup.py object
  """
  # A placeholder in case something should be added to the footer of all the
  # sub pages.
  return page

# ***************************************************************************
def write_sub_page(opts,section,html_sections,style,script):
  """ 
  A function to write each of the individual sections into a markup.py object.

  @param opts: UNDOCUMENTED
  @param section: The section being generated
  @param html_sections : A list of section titles
  @param style: The css style file sent as location of this file
  @param script: A dict of the java scripts used by the html.  
  """
  global h2_num
  global h3_num
  print "...Processing " + section
  subPage = initialize_page(section,style,script)
  subPage.p(e.br())
  subPage.add("<!-- beginning of a sub section -->")
  subPage.div(class_="contenu")
  id = section.replace(" ","_")
  h2_num += 1
  subPage.h2(str(h2_num) + ' ' + html_sections[section])
  logText(logfile, html_sections[section], "section")
  h3_num = 1
  subPage = write_results(subPage, opts, section)
  subPage.div.close()
  subPage.add("<!-- close div contenu-->")
  subPage.div.close()
  add_page_footer(subPage)
  subHtmlFile = file(opts.physdir + '/' + section + '.html',"w") 
  subHtmlFile.write(subPage(False))
  subHtmlFile.close
  mainpage.div(class_="menuitem")
  mainpage.add('\t<a class="menulink" href="javascript:loadFrame(\'' + \
      section + '.html\');"> ' + str(h2_num) + ': ' + \
      html_sections[section] + '</a>\n' )
  mainpage.div.close()
  return subPage

# ***************************************************************************
def functionId(nFramesUp):
  """ 
  Create a string naming the function n frames up on the stack.
  
  @param nFramesUp: input (type: number)
  @return: message
  """
  try:
    co = sys._getframe(nFramesUp+1).f_code
    msg = "%s (%s @ %d)" % (co.co_name, co.co_filename, co.co_firstlineno)
    if msg.startswith("?") is False:
      print "-->ERROR in function: " + msg
  except:
    msg="?"

  return msg

# ***************************************************************************
def logText(logfile, text, tag="done"):
  """
  Utility to populate a logfile in HTML format. 
  The third argument is a string. Depending on its value, the text will have 
  different color. 

  @param logfile: UNDOCUMENTED
  @param text: a text to be printed (type: string)
  @param tag: is in {"done","warning","error"}  (type: string )
  """ 

  if tag=="warning":
    msg= "<"+tag+">"+text+"</"+tag+">\n"
    logfile.write(msg)
    if opts.verbose is True:
      print >>sys.stdout,text
  elif tag=="error":
    msg = "<"+tag+">"+text
    logfile.write(msg)
    i =1 
    while len(msg)>0:
      msg = functionId(i)
      if msg.startswith("?") is False:
        logfile.write("\n"+msg)
      else :
        logfile.write("</"+tag+">\n")
        msg=[]
      i=i+1
    print >>sys.stderr, text
  else :
    msg = "<"+tag+">"+text+"</"+tag+">\n"
    logfile.write( msg )
    if opts.verbose is True:
      print >>sys.stdout,text

# ***************************************************************************
def patternFoundInFilename(filename, pattern):
  """
   a small function to check that a pattern is contained within a filename.

     >>> filename = "H1H2L1-plotinspmissed_totalMass_versus_eta.png"
     >>> pattern = "plotinspmissed*eta"
     >>> patternfoundInFilename(filename, pattern)

   @param filename: the filename to look at (type: string)
   @param pattern: a pattern which may contain several '*' (type: string)
   @return: boolean
  """
  tokeep = False
  if pattern is None : return tokeep
  
  # you may have * inside a pattern
  for word in pattern.split('*'):
    if word in filename:
      tokeep = True
    else:
      tokeep = False
      # if one word is not found within filename, we must break and se
      break

  return tokeep

# ***************************************************************************
def make_external_call(\
  command, show_stdout=False, \
  show_command=False, show_error=True, return_stderr=False):
  """
  Run a command line argument and print informative messages on failure.
  It returns two outputs: the stdout of the command, and its status.  

    >>> make_external_call('cp * /tmp', False, False, True)

  @param command: the command to try (type: string)
  @param show_stdout: show the stdout  (type: boolean)
  @param show_command: show the command (type: boolean)
  @param show_error: show the error if any (type: boolean)
  @return: the stdout and a status  


  """
  if show_command and opts.verbose is True: 
    print "--- Trying this command :" + command

  stdin, out, err = os.popen3(command)
  pid, status = os.wait()
  this_output = out.read()
  if show_error & status != 0:
    print >>sys.stderr, "External call failed."
    print >>sys.stderr, "  status: %d" % status
    print >>sys.stderr, "  stdout: %s" % this_output
    print >>sys.stderr, "  stderr: %s" % err.read()
    print >>sys.stderr, "  command: %s" % command
    sys.exit(status)
  if show_stdout:
    if this_output[0:1]=='\n': 
      print  this_output[1:]  #first character is \n
    else:
      print this_output

  this_error = err.read()

  stdin.close()
  out.close()
  err.close()  
  if return_stderr:
    return this_output, this_error, status
  else:
    return this_output, status

# ***************************************************************************
def mkdir( newdir ):
  """
  Create a directory

  @param newdir : name of directory to be created (type: string)
  """
  if os.path.isdir(newdir): 
    print >>sys.stdout,"WARNING: this directory already exists (" + newdir +")."
    pass
  elif os.path.isfile(newdir):
    raise OSError("a file with the same name as the desired " \
                  "dir, '%s', already exists." % newdir)
  else: 
    os.mkdir(newdir)


# ***************************************************************************
def create_toggle():
  """
  This function is just an alias to create a javascript for the toggle on/off. 

  @return: nothing
  """
  fname = open("toggle.js", "w")
  fname.write("""function afterLoadFrame() {
    $('#iframecontent h3').click(function() { $(this).next().toggle('fast'); });
    $('#iframecontent a[rel="external"]').attr('target','_blank');
    $('#iframecontent input').hide();
    $('#iframecontent p:first').hide();
    }
  function loadFrame(sourceURL) {
    $("#iframecontent").load(sourceURL,{},afterLoadFrame);
    /* Remove the last two arguments to disable toggling from the title. */
    }
""")
  fname.close()

# ***************************************************************************
def write_results(page, opts, section,injection=None):
  """
  This function is just a switch to the function that create each section of
  the HTML document. 
   
    >>> write_results(page, opts, "injection")

  @param page: the HTML document page. (type: markup.py document)
  @param opts: the main options variable (type: write_ihope_page options)
  @param section: the current section to switch to  (type: string)
  @return: an update of HTML document
  @param injection: The name of the injection when section=injection
  """

  if opts.verbose is True:
    print >>sys.stdout,"--------------------- Creating section "+section
  if section=='general':
    page = write_general(page, opts)
  elif section=='datainfo':
    page = write_datainfo(page, opts)
  elif section=='playsummary':
    page = write_summary(page,opts,thisSearch='playground',injection='allinj',
                         coinc_plots='coinc_result_plots')
  elif section=='fullsummary':
    page = write_summary(page,opts,thisSearch='full_data',injection='allinj',
                         coinc_plots='coinc_result_plots')
  elif section=='play_full_slide_summary':
    page = write_summary(page,opts,thisSearch='full_data_slide',
                         injection='allinj', coinc_plots='coinc_result_plots')
  elif section=='playground': 	
    page = write_analysis(page, opts,thisSearch='playground',
                          coinc_plots='coinc_result_plots')
  elif section=='full_data_slide': 	
    page = write_analysis(page, opts,thisSearch='full_data_slide',
                          coinc_plots='coinc_result_plots')
  elif section in hipecp.options("injections"):
    page = write_injection(page, opts,section, coinc_plots='coinc_result_plots')
  elif section=='allinj':
    page = write_injection(page, opts,section, coinc_plots='coinc_result_plots')
  elif section=='hardware_injections':
    page = write_hw_injection(page, opts)
  elif section=='full_data':
    page=write_analysis(page,opts,thisSearch='full_data',
                        coinc_plots='coinc_result_plots')
  elif section=='upperlimit_play': 	
    page = write_upperlimit(page, opts,thisSearch='playground',
                            coinc_plots='coinc_result_plots')
  elif section=='upperlimit_full':
    page = write_upperlimit(page, opts,thisSearch='full_data',
                            coinc_plots='coinc_result_plots')
  elif section=='logfile':
    page = write_logfile(page, opts)
  elif section=='about': 
    page = write_about(page, opts)
  else:
    msg = "this section ("+ section +") does not exist. "
    logText(logfile, msg, "error")

  return page


# ***************************************************************************
def write_logfile(page , opts):
  """
  This function creates an entry with the logfile information.
  
  @param page: the HTML document page. (type: markup.py document)
  @param opts: the main options variable (type: write_ihope_page options)
  @return: an update of HTML document
  """
  # get the directory of the url
  dir = opts.webdir +'/'
  
  source = dir + __name__ + '.xml'
  
  page.add("<p> Here is a logfile.")
  page = add_href(page, href=source, name=__name__)
  page.add(", which entry should be all in green except if skip options were used.</p>")
  
  return page

def add_href(page, href="", name="click here"):
  try:
    page.add("<a href=\"" + href + "\" rel=\"external\"> " + name + "</a>")
  except: 
    logText(logfile, "Could not create link "+ href, "error")
  return page

# ***************************************************************************

def create_sections(page,opts,sections,tags,captions,imagetags,images_dirs,\
                    titles,comments,configs,htmltags,htmlOnly,htmlAndPlot,\
                    allIfosFlag):
  for section in sections:
      if section in allIfosFlag:
        allIfos = True
      else:
        allIfos = False
      page = heading(page, titles[section])
      if comments[section] is not None:
        page.p(comments[section])
      page = add_config_section(page, configs[section])
      if section in htmlOnly:
        page = html_insert(page,opts, html_tag=htmltags[section],\
          caption=captions[section],directory=images_dirs[section],\
          all_ifos = allIfos)
      elif section in htmlAndPlot:
        page = html_and_plot(page, opts, cachefile_tag=tags[section],\
          caption=captions[section],image_tag=imagetags[section],\
          directory=images_dirs[section],html_tag=htmltags[section],\
          all_ifos = allIfos)
      else:
        page = fom(page, opts, cachefile_tag=tags[section],\
          caption=captions[section],image_tag=imagetags[section],\
          directory=images_dirs[section],all_ifos=allIfos)
      page.div.close() # for each heading

  return page



# ***************************************************************************
def write_general(page,opts):
  """
  Creates the general section. 

  @param page: the HTML document page. (type: markup.py document)
  @param opts: the main options variable (type: write_ihope_page options)
  @return: an update of HTML document
  """
  webdir = opts.webdir
  ini = opts.config_file
    
  text = 'This page summarizes the analysis of the data segment from GPS '
  text += 'time %s up to %s' % (opts.gps_start_time,  opts.gps_end_time)
  page.add(text)

  try:
    # let us give the *.ini and log files.
    # we also need to copy some files to the webdir
    cmd = 'cp '+opts.datadir + '/*.pipeline.log  '+opts.physdir
    make_external_call(cmd, opts.debug, opts.debug, True)
    cmd = 'cp '+opts.datadir + '/*.ini  '+opts.physdir
    make_external_call(cmd, opts.debug, opts.debug, True)
    iniFiles = glob.glob( opts.physdir + '/*.ini' )
  except:pass

  heading(page, "Configuration files:")
  try:
    page.add("The configuration parameters are contained within the file(s) ")
    page.ul()
    for file in iniFiles:
      cmd = 'mv ' + file + ' ' + file + '.txt'
      make_external_call(cmd, opts.debug, opts.debug, True)
      logname = file.replace( opts.physdir, '' )
      page.li( e.a( logname, href=webdir + "/" + logname + '.txt', rel="external" ) )
    page.ul.close()
  except:pass

  page.ol.close() # end enumerate

  logText(logfile,  "...Get the executables version...")
  page.div.close()

  # Do all the libraries.
  # Collect information
  library_list = get_library_version_info()

  for entry in library_list:
    page = heading(page, entry['Name'] + " versioning information.")
    page.ul()
    page.li("Git hash tag: %s" %(entry['ID'],))
    page.li("Status: %s" %(entry['Status'],))
    page.li("Branch: %s" %(entry['Branch'],))
    page.li("Version number: %s" %(entry['Version'],))
    page.li("Tag (if any): %s" %(entry['Tag'],))
    page.li("Author: %s" %(entry['Author'],))
    page.li("Committer: %s" %(entry['Committer'],))
    if entry.has_key('Builder'):
      page.li("Builder: %s" %(entry['Builder'],))
    page.li("Date: %s" %(entry['Date'],))
    page.ul.close()
    page.div.close()

  # List of executables
  if opts.verbose is True: 
    print "Extracting the version and tag of the executables..." 
  
  page = heading(page, "Version information from executables.")
  code_version_dict = get_code_version_numbers(hipecp)

  page.ol() #starts enumerate
  # first section with version number
  # get the version of each executables
  try:
    page.ul()
    for exe, message in code_version_dict.items():
      text = "<b>" + exe + "</b>: " + message
      page.li(text)   
    page.ul.close()
    page.li.close()
  except:
    logText(logfile, """Problem with the executable: cannot find them ? """, "warning")
    pass
  page.div.close()
  
  # The ifo requested
  page = heading(page, "This search concerned the following combination of ifos")
  page = add_config_section(page, "workflow-ifos")
  page.div.close() #close the main heading

  return page


# ***************************************************************************
def write_table(page, segs, keys,printKeys = True):
  """
  print a table in html format, where keys is the first column and 
  keys to the dictionary "segs"

  @param page: the html document
  @param segs: the segment durations (type: a dictionary which keys are stored in the parameter "keys")
  @param keys: the segments names (type: a list of sorted keys corresponding to the segs dictionary keys (not sorted))
  @param printKeys: UNDOCUMENTED
  """
  page.table()
  for key in keys:
    if segs.has_key(key):
      page.tr()
      if printKeys:
        page.td(key)
      page.td(segs[key])
      page.tr.close()
  page.table.close()

  return page
# ***************************************************************************
def write_summary(page,opts,thisSearch='playground',injection = 'allinj',
                  coinc_plots = 'coinc_result_plots'):
  """
  Creates the summary section.

  @param page: the HTML document page. (type: markup.py document)
  @param opts: the main options variable (type: write_ihope_page options)
  @param thisSearch: either "playground" or "full_data" or "full_data_slide" (type: string)
  @param injection: UNDOCUMENTED
  @param coinc_plots: UNDOCUMENTED
  @return: an update of HTML document
  """
  webdir = opts.webdir
  ini = opts.config_file
  ifos = get_ifos()
  msg = "This section summarizes the analysis of the "+thisSearch+" data.<br/>"
  page.add(msg)
  if thisSearch=='playground':
    ZLdir='playground_summary_plots/'
    ifartag=['PLAYGROUND','PLAYGROUND','CLOSED_BOX']
  elif thisSearch=='full_data_slide':
    ZLdir='full_data_slide_summary_plots/'
    ifartag=['FULL_DATA','PLAYGROUND','CLOSED_BOX']
  elif thisSearch=='full_data':
    ZLdir='full_data_summary_plots/'
    ifartag=['FULL_DATA','ALL_DATA','OPEN_BOX']
  else:
    print 'Error in write_analysis function: thisSearch is not valid'
    sys.exit(1)
  IJdir = injection + '_summary_plots/'
  PDdir = coinc_plots + '/'
  if opts.symlink_plots:
    symlink_plot_directory(opts.datadir + ZLdir,\
        opts.physdir + ZLdir)
  else:
    copy_plot_directory(opts.datadir + ZLdir,\
        opts.physdir + ZLdir)
  if opts.symlink_plots:
    symlink_plot_directory(opts.datadir + IJdir,\
        opts.physdir + IJdir)
  else:
    copy_plot_directory(opts.datadir + IJdir,\
        opts.physdir + IJdir)
  if opts.symlink_plots:
    symlink_plot_directory(opts.datadir + PDdir,\
        opts.physdir + PDdir)
  else:
    copy_plot_directory(opts.datadir + PDdir,\
        opts.physdir + PDdir,open_box = opts.full_data)

  # A keyword to identify the section
  # title will be the name of the section.
  # tags is a tag to search for the relevant cache file (will use the first one found)
  # imagetags. if not none, will only pu a subset of images corresponding to the list provided. 
  # captions is the text to be written in the caption
  # Comments: If not none a comment will be appended to the web page
  # configs : is not none, the part of the ini file corresponding to this config name will be written in the web page
  # images_dirs : What directory to look in.

  sections = []
  htmlOnly = []
  htmlAndPlot = []
  allIfosFlag = []
  titles = {}
  tags = {}
  htmltags = {}
  imagetags = {}
  captions = {}
  comments = {}
  configs = {}
  images_dirs = {}

  section = 'inspiralrange'
  sections.append(section)
  allIfosFlag.append(section)
  titles[section] = "Inspiral range plots"
  tags[section] = '*inspiralrange*'
  imagetags[section] = ['range_plot']
  captions[section] = """Inspiral Horizon distance for a \
       (1.4,1.4) solar mass system with SNR=8 (first sub-figure)."""
  comments[section] = None
  configs[section] = None
  images_dirs[section] = ZLdir

  section = 'ifar2'
  sections.append(section)
  htmlAndPlot.append(section)
  titles[section] = "IFAR and loudest event tables with CAT2 vetoes applied (these include the CBC hardware injections)."
  tags[section] = '*plotifar*'+ifartag[0]+'*CAT_2*'+ifartag[1] + '*'
  htmltags[section] = '*'+ifartag[0]+'*CAT_2_VETO_LOUDEST*'+ifartag[1]+'*_EVENTS_BY_COMBINED_FAR*'
  imagetags[section] = ['cumhist_combined_ifar']
  captions[section] = "Combined and uncombined IFAR vs cumulative number of triggers (above). Loudest event tables as a function of combined FAR (below)."
  comments[section] = None
  configs[section] = None
  images_dirs[section] = PDdir

  section = 'ifar3'
  sections.append(section)
  htmlAndPlot.append(section)
  titles[section] = "IFAR and loudest event tables with CAT2 vetoes applied and CBC hardware injections removed."
  tags[section] = '*plotifar*'+ifartag[0]+'*CAT_3*'+ifartag[1] + '*'
  htmltags[section] = '*'+ifartag[0]+'*CAT_3_VETO_LOUDEST*'+ifartag[1]+'*_EVENTS_BY_COMBINED_FAR*'
  imagetags[section] = ['cumhist_combined_ifar']
  captions[section] = "Combined and uncombined IFAR vs cumulative number of triggers (above). Loudest event tables as a function of combined FAR (below)."
  comments[section] = None
  configs[section] = None
  images_dirs[section] = PDdir

  section = 'ifar4'
  sections.append(section)
  htmlAndPlot.append(section)
  titles[section] = "IFAR and loudest event tables with CAT2 and CAT3 vetoes applied and CBC hardware injections removed."
  tags[section] = '*plotifar*'+ifartag[0]+'*CAT_4*'+ifartag[1] + '*'
  htmltags[section] = '*'+ifartag[0]+'*CAT_4_VETO_LOUDEST*'+ifartag[1]+'*_EVENTS_BY_COMBINED_FAR*'
  imagetags[section] = ['cumhist_combined_ifar']
  captions[section] = "Combined and uncombined IFAR vs cumulative number of triggers (above). Loudest event tables as a function of combined FAR (below)."
  comments[section] = None
  configs[section] = None
  images_dirs[section] = PDdir

  section = 'inspmissed'
  sections.append(section)
  htmlAndPlot.append(section)
  titles[section] = "Found and missed plots (second stage) and the 10 closest missed injections (CAT2 and CAT3 vetoes applied and CBC hardware injections removed)."
  tags[section] = '*ligolw_cbc_plotfm_fm_dist_v_param_ALLINJ*CAT_4*'
  htmltags[section] = '*'+injection.upper()+'_CAT_4_VETO_CLOSEST_MISSED_INJECTIONS_SUMMARY*'
  imagetags[section] = ['cbc_plotfm_fm_dist_v_param']
  captions[section] = 'Found and missed injections. Stars are injections found with zero combined far; circles are injections found with non-zero combined far (their color corresponds to their far). Red crosses are missed injections; vetoed injections are excluded. The decisive distance is the second smallest effective distance. The table below shows the 10 closest missed injections (i.e., the 10 closest red crosses). For a table of the non-zero FAR injections (the colored circles), please go to the "All Injections Combined" section and click on "Loudest injections that are not louder than the loudest slide."'
  comments[section] = None
  configs[section] = None
  images_dirs[section] = PDdir
  
  page = create_sections(page,opts,sections,tags,captions,imagetags,\
                    images_dirs,titles,comments,configs,htmltags,htmlOnly,\
                    htmlAndPlot,allIfosFlag)

  return page




# ***************************************************************************
def write_datainfo(page,opts):
  """
  Creates the datainfo section.

  @param page: the HTML document page. (type: markup.py document)
  @param opts: the main options variable (type: write_ihope_page options)
  @return: an update of HTML document
  """
  # first, get some information 
  webdir = opts.webdir
  datadir = opts.datadir
  ini = opts.config_file
  ifos = get_ifos()

  page.add("<p>This section summarizes the data which was available for analysis.")
  page.add("All of the segment files are linked in")
  page = add_href(page, href= webdir + "segments", name="this directory")
  page.add(".<p>")

  page.add("Segments were generated using the following information:")
  page = add_config_section(page, "workflow-segments")

  # Start with the single detector times
  ifos = get_ifos()
  ifos.sort()
  ifos = ''.join(ifos)
  file_name = ifos + '-WORKFLOW_SUMMARY-' + opts.gps_start_time + '-' + \
              opts.duration + '.xml'
  seg_summary_file = datadir + "/summary/" + file_name
  descript_dict = {}
  descript_dict["CBC_WORKFLOW_SCIENCE"] = "Single detector ANALYSIS_READY " +\
                                          "times."
  descript_dict["CBC_WORKFLOW_SCIENCE_OK"] = "Single detector ANALYSIS_READY "+\
                     "with CAT_1 vetoed time removed."
  descript_dict["CBC_WORKFLOW_SCIENCE_AVAILABLE"] = "Single detector " +\
                            "ANALYSIS_READY times present on cluster " +\
                            "with CAT_1 vetoed time removed."
  descript_dict["CBC_WORKFLOW_ANALYSABLE"] = "Single detector times that " +\
                           "were analysed after missing data, CAT_1 vetoes " +\
                           "and analysis buffers are removed."
  descript_dict["ANALYSABLE_CAT_2"] = "Single detector times that " +\
                           "were analysed after missing data, CAT_2 vetoes " +\
                           "and analysis buffers are removed."
  descript_dict["ANALYSABLE_CAT_3"] = "Single detector times that " +\
                           "were analysed after missing data, CAT_3 vetoes " +\
                           "and analysis buffers are removed."
  descript_dict["ANALYSABLE_CAT_4"] = "Single detector times that " +\
                           "were analysed after missing data, CAT_4 vetoes " +\
                           "and analysis buffers are removed."
  descript_dict["ANALYSABLE_CAT_5"] = "Single detector times that " +\
                           "were analysed after missing data, CAT_5 vetoes " +\
                           "and analysis buffers are removed."
  # Order is important here, so I don't just do descript_dict.keys()
  previous_list = None
  for tag in ["CBC_WORKFLOW_SCIENCE","CBC_WORKFLOW_SCIENCE_OK",
              "CBC_WORKFLOW_SCIENCE_AVAILABLE", "CBC_WORKFLOW_ANALYSABLE",
              "ANALYSABLE_CAT_2", "ANALYSABLE_CAT_3", "ANALYSABLE_CAT_4",
              "ANALYSABLE_CAT_5"] :
    page = heading(page, descript_dict[tag])
    if tag == "CBC_WORKFLOW_SCIENCE_OK":
      msg = '<span style=\"color:red\">'
      msg += "Other options that remove data from the workflow-segments "
      msg += "section are removed from this list. This includes the minimum "
      msg += "segment length option."
      msg += "</span>"
      page.add(msg)
    elif tag == "CBC_WORKFLOW_SCIENCE_AVAILABLE":
      msg = '<span style=\"color:red\">'
      msg += "Other options that remove data from the workflow-datafind "
      msg += "section are removed from this list. "
      msg += "</span>"

    try:
      if tag.startswith("ANALYSABLE_CAT_"):
        segs_dict = {}
        segs_dict_analysable = get_segs_from_file(seg_summary_file,
                                       "CBC_WORKFLOW_ANALYSABLE", version=None)
        veto_file_name = ifos+'-CUMULATIVE_CAT_%s_VETO_SEGMENTS.xml' %(tag[-1])
        veto_file = datadir + "/segments/" + veto_file_name
        veto_times = get_segs_from_file(veto_file,
                                            "VETO_CAT%s_CUMULATIVE" %(tag[-1]))
        for ifo in segs_dict_analysable.keys():
          unvetoed = segs_dict_analysable[ifo].active - veto_times[ifo].active
          curr_seg = segment_utils.LigolwSegmentList(active=unvetoed,
                                                      instruments = set([ifo]))
          segs_dict[ifo] = curr_seg
      else:
        segs_dict = get_segs_from_file(seg_summary_file, tag, version=None)
      this_data = {}
      this_data['ifo'] = ['duration(days)' ,'duration (s)']
      keys = ["ifo"] + segs_dict.keys()
      for key in segs_dict.keys():
        curr_ifo = ''.join(segs_dict[key].instruments)
        time = abs(segs_dict[key].active)
        this_data[curr_ifo] = [str(round((float(time)/86400.),3)),
                               str(float(time))]
      page = write_table(page, this_data, keys)

      if previous_list is None:
        msg = "Complete set of ANALYSIS READY times."
        page.add(msg)
        this_data = {}
        this_data['ID'] = ['IFO', 'start time', 'end_time']
        keys = ['ID']
        count = 0
        for ifo in segs_dict.keys():
          for seg in segs_dict[ifo].active:
            this_data[str(count)] = [ifo, seg[0], seg[1]]
            keys.append(str(count))
            count = count+1
        page = write_table(page, this_data, keys, printKeys = False)
        previous_list = segs_dict
      else:
        msg = "Time removed compared with previous section." 
        page.add(msg)

        this_data = {}
        this_data['ID'] = ['IFO', 'start time', 'end_time']
        keys = ['ID']
        count = 0
        for ifo in previous_list.keys():
          segs_diff = previous_list[ifo].active - segs_dict[ifo].active
          for seg in segs_diff:
            this_data[str(count)] = [ifo, seg[0], seg[1]]
            keys.append(str(count))
            count = count+1
        page = write_table(page, this_data, keys, printKeys = False)
        previous_list = segs_dict
    except:
      msg = "Problems parsing category veto segment list "+tag
      logText(logfile, msg,"warning")
    page.div.close()

  i=0
  catStrings = ["VETOTIME_CAT1"]
  categories = (hipecp.get('segments','veto-categories')).split(',')[:-1]
  for num in categories:
    catStrings.append("VETOTIME_CAT" + str(num))
  for cat in (catStrings):
    i=i+1
    msg = "Coincidence times after category " +str(i) + " veto segments are "+\
          "applied."
    page = heading(page, msg) 

    msg = "Coincident live times (only available if workflow has finished " +\
          "and ran at this veto category)."
    page.add(msg)
    key = 0
    keys = []
    data = {}
    keys.append(key)
    data[key] = ['Coincidence','Coincidence time(days)','Percent of duration']
    totTime = 0
    for coinc in get_ifo_coinc():
      key += 1
      summGlob = datadir + '/coinc_result_plots/' + coinc + '-FULL_DATA'
      if i != 1:
        summGlob += '_CAT_' + str(i) + '_VETO'
      summGlob += '_LOUDEST_ALL_DATA_EVENTS_BY_COMBINED_FAR_SUMMARY-*.xml'
      summFiles = glob.glob(summGlob)
      if len(summFiles) == 1:
        xmldoc = utils.load_filename(summFiles[0], contenthandler=ContentHandler)
        loudest_event_table = table.get_table(xmldoc, 'loudest_events')
        if len (loudest_event_table):
          time = loudest_event_table[0].all_data_duration__Px_days_xP_
        else:
          time = 0
        keys.append(key)
        data[key] = [coinc,str(round(time,3)),str(round(time*100/opts.daydur,3))]  
        totTime += time
    key += 1
    if totTime != 0:
      keys.append(key)
      data[key] = ['Total',str(round(totTime,3)),str(round(totTime*100/opts.daydur,3))]
    page = write_table(page, data,keys,printKeys = False)
    for iterafsaf in [1]:
      msg = "This category includes the following flags : "
      page.add(msg)
      key = 0
      keys = []
      formFlags = {}
      keys.append(str(key))
      formFlags[str(key)] = ['IFO','Flag','Version','Start padding (s)', 'End padding (s)','Start time','End time']
      key += 1
      seg_filename = hipecp.get("workflow-segments", "segments-veto-definer-file")
      xmldoc = utils.load_filename( seg_filename, gz = seg_filename.endswith(".gz"), contenthandler=ContentHandler )
      veto_def_tab = table.get_table(xmldoc,'veto_definer')
      output = StringIO.StringIO()
      tableList = ['veto_definer']
      columnList = ['ifo', 'name', 'version', 'start_pad', 'end_pad', 'category', 'start_time', 'end_time']
      print_tables.print_tables(xmldoc, output, 'wiki', tableList = tableList, columnList = columnList,
        round_floats = False, format_links = False, print_table_names = False )
      flags = output.getvalue().split('\n') 
      output.close()
      for ifo in get_ifos():
        for line in flags:
          if line == '' or line == '||ifo||name||version||start pad||end pad||category||start time||end time||':
            continue
          flag = line.split('||')
          flag.pop(0)
          flag.pop(-1)
          if int(flag[5]) != i:
            continue
          if flag[0] != ifo:
            continue
          flag.pop(6)
          keys.append(str(key))
          formFlags[str(key)] = flag
          key += 1

      page = write_table(page, formFlags,keys,printKeys = False)
    page.div.close()

  return page


# ***************************************************************************
def write_upperlimit(page, opts, thisSearch='playground',
                     coinc_plots='coinc_result_plots'):
  """
  Creates an upper limit section
  
  @param page: the HTML document page. (type: markup.py document)
  @param opts: the main options variable (type: write_ihope_page options)
  @param thisSearch: either "playground" or "full_data" or "full_data_slide" (type: string)
  @param coinc_plots: UNDOCUMENTED
  @return: an update of HTML document
  """
  webdir = opts.webdir
  ini = opts.config_file
  ifos = get_ifos()

  if thisSearch=='playground':
    ifartag=['PLAYGROUND','PLAYGROUND','CLOSED_BOX']
    FARtext="""a user-specified FAR value (1/livetime by default)."""
  elif thisSearch=='full_data_slide':
    ifartag=['FULL_DATA','PLAYGROUND','CLOSED_BOX']
    FARtext="""an unknown FAR: upper limit for full data slide is not well-defined!""" #HERE BE DRAGONS
  elif thisSearch=='full_data':
    ifartag=['FULL_DATA','ALL_DATA','OPEN_BOX']
    FARtext="""the combined FAR of the loudest full data event."""
  else:
    print 'Error in write_analysis function: thisSearch is not valid'
    sys.exit(1)
  PDdir = coinc_plots + '/'

  page.add("This section gives the search sensitivity and upper limits using "+FARtext)
  page.add("<br/>")

  if opts.symlink_plots:
    symlink_plot_directory(opts.datadir + PDdir,\
        opts.physdir + PDdir)
  else:
    copy_plot_directory(opts.datadir + PDdir,\
        opts.physdir + PDdir,open_box = opts.full_data)

  sections = []
  htmlOnly = []
  htmlAndPlot = []
  allIfosFlag = []
  titles = {}
  tags = {}
  htmltags = {}
  imagetags = {}
  captions = {}
  comments = {}
  configs = {}
  images_dirs = {}

  section= 'upperlimitTotalMassRangeplots'
  sections.append(section)
  htmlAndPlot.append(section)
  titles[section] = "Sensitive Distance as a Function of Total Mass"
  tags[section] = '*sink_by_total_mass_'+ifartag[0]+'_CAT_4_VETO*'
  htmltags[section] = '*sink_by_total_mass_combined_upper_limit_'+ifartag[0]+'_CAT_4_VETO*'
  imagetags[section] = ['distance']
  captions[section] = """Mean sensitive distance for the CBC search pipeline as a function of total mass. The sensitive distance is determined by computing the cube root of the mean distance^3 for injections found with a FAR below """+FARtext+""" (CAT2 and CAT3 vetoes applied and CBC hardware injections removed)."""
  comments[section] = None
  configs[section] = 'search-volume'
  images_dirs[section] = PDdir

  section= 'upperlimitChirpMassRangeplots'
  sections.append(section)
  htmlAndPlot.append(section)
  titles[section] = "Sensitive Distance as a Function of Chirp Mass"
  tags[section] = '*sink_by_chirp_mass_'+ifartag[0]+'_CAT_4_VETO*'
  htmltags[section] = '*sink_by_chirp_mass_combined_upper_limit_'+ifartag[0]+'_CAT_4_VETO*'
  imagetags[section] = ['distance']
  captions[section] = """Mean sensitive distance for the CBC search pipeline as a function of chirp mass. The sensitive distance is determined by computing the cube root of the mean distance^3 for injections found with a FAR below """+FARtext+""" (CAT2 and CAT3 vetoes applied and CBC hardware injections removed)."""
  comments[section] = None
  configs[section] = 'search-volume'
  images_dirs[section] = PDdir

  section= 'upperlimitM1M2Rangeplots'
  sections.append(section)
  htmlAndPlot.append(section)
  titles[section] = "Sensitive Distance as a Function of Component Masses."
  tags[section] = '*sink_by_component_mass_'+ifartag[0]+'_CAT_4_VETO*'
  htmltags[section] = '*sink_by_component_mass_combined_upper_limit_'+ifartag[0]+'_CAT_4_VETO*'
  imagetags[section] = ['distance']
  captions[section] = """Mean sensitive distance for the CBC search pipeline as a function of the component masses. The sensitive distance is determined by computing the cube root of the mean distance^3 for injections found with a FAR below """+FARtext+""" (CAT2 and CAT3 vetoes applied and CBC hardware injections removed)."""
  comments[section] = None
  configs[section] = 'search-volume'
  images_dirs[section] = PDdir


  section= 'combupperlimitVmtotal'
  sections.append(section)
  htmlAndPlot.append(section)
  titles[section] = "Combined Upper Limits as a Function of Total Mass."
  tags[section] = '*sink_by_total_mass_'+ifartag[0]+'_CAT_4_VETO*'
  htmltags[section] = '*sink_by_total_mass_combined_upper_limit_'+ifartag[0]+'_CAT_4_VETO*'
  imagetags[section] = ['upper_limit_plot']
  captions[section] = """Combined upper limits on the CBC rate (in mergers/Mpc^3/yr) as a function of total mass. The plot shows the 90&#37; upper limits arising both from the prior and the posterior (CAT2 and CAT3 vetoes applied and CBC hardware injections removed)."""
  comments[section] = None
  configs[section] = 'upper-limit'
  images_dirs[section] = PDdir

  section= 'combupperlimitVmchirp'
  sections.append(section)
  htmlAndPlot.append(section)
  titles[section] = "Combined Upper Limits as a Function of Chirp Mass."
  tags[section] = '*sink_by_chirp_mass_'+ifartag[0]+'_CAT_4_VETO*'
  htmltags[section] = '*sink_by_chirp_mass_combined_upper_limit_'+ifartag[0]+'_CAT_4_VETO*'
  imagetags[section] = ['upper_limit_plot']
  captions[section] = """Combined upper limits on the CBC rate (in mergers/Mpc^3/yr) as a function of chirp mass. The plot shows the 90&#37; upper limits arising both from the prior and the posterior (CAT2 and CAT3 vetoes applied and CBC hardware injections removed)."""
  comments[section] = None
  configs[section] = 'upper-limit'
  images_dirs[section] = PDdir

  section= 'combupperlimitVm1m2'
  sections.append(section)
  htmlAndPlot.append(section)
  titles[section] = "Combined Upper Limits as a Function of Component Masses."
  tags[section] = '*sink_by_component_mass_'+ifartag[0]+'_CAT_4_VETO*'
  htmltags[section] = '*sink_by_component_mass_combined_upper_limit_'+ifartag[0]+'_CAT_4_VETO*'
  imagetags[section] = ['upper_limit_plot']
  captions[section] = """Combined upper limits on the CBC rate (in mergers/Mpc^3/yr) as a function of the component masses. The plot shows the 90&#37; upper limits arising both from the prior and the posterior (CAT2 and CAT3 vetoes applied and CBC hardware injections removed)."""
  comments[section] = None
  configs[section] = 'upper-limit'
  images_dirs[section] = PDdir

  page = create_sections(page,opts,sections,tags,captions,imagetags,\
                    images_dirs,titles,comments,configs,htmltags,htmlOnly,\
                    htmlAndPlot,allIfosFlag)

  return page

#***************************************************************************


def write_analysis(page, opts, thisSearch='playground',
                   coinc_plots='coinc_result_plots'):
  """
  Creates the playground or full_data section. It uses the same function 
  because except the name and time analysed, the figures of merits are the same. 
  @param page: the HTML document page. (type: markup.py document)
  @param opts: the main options variable (type: write_ihope_page options)
  @param thisSearch: either "playground" or "full_data" or "full_data_slide" (type: string)
  @param coinc_plots: UNDOCUMENTED
  @return: an update of HTML document
  """
  webdir = opts.webdir
  ini = opts.config_file
  ifos = get_ifos()
  msg = "This section summarizes the analysis of the "+thisSearch+" data.<br/>"
  page.add(msg)
  if thisSearch=='playground':
    images_dir='playground_summary_plots/'
    ifartag=['PLAYGROUND','PLAYGROUND','CLOSED_BOX']
  elif thisSearch=='full_data_slide':
    images_dir='full_data_slide_summary_plots/'
    ifartag=['FULL_DATA','PLAYGROUND','CLOSED_BOX']
  elif thisSearch=='full_data':
    images_dir='full_data_summary_plots/'
    ifartag=['FULL_DATA','ALL_DATA','OPEN_BOX']
  else:
    print 'Error in write_analysis function: thisSearch is not valid'
    sys.exit(1)
  PDdir = coinc_plots + '/'
  if opts.symlink_plots:
    symlink_plot_directory(opts.datadir + images_dir,\
                      opts.physdir + images_dir)
  else:
    copy_plot_directory(opts.datadir + images_dir,\
                      opts.physdir + images_dir)
  if opts.symlink_plots:
    symlink_plot_directory(opts.datadir + PDdir,\
        opts.physdir + PDdir)
  else:
    copy_plot_directory(opts.datadir + PDdir,\
        opts.physdir + PDdir,open_box = opts.full_data)

  # A keyword to identify the section
  # title will be the name of the section.
  # tags is a tag to search for the relevant cache file (will use the first one found)
  # imagetags. if not none, will only pu a subset of images corresponding to the list provided. 
  # captions is the text to be written in the caption
  # Comments: If not none a comment will be appended to the web page
  # configs : is not none, the part of the ini file corresponding to this config name will be written in the web page
  # images_dirs : What directory to look in.

  sections = []
  htmlOnly = []
  htmlAndPlot = []
  allIfosFlag = []
  titles = {}
  tags = {}
  htmltags = {}
  imagetags = {}
  captions = {}
  comments = {}
  configs = {}
  images_dirs = {}

  section = 'inspiralrange'
  sections.append(section)
  allIfosFlag.append(section)
  titles[section] = "Inspiral range plots"
  tags[section] = '*inspiralrange*'
  imagetags[section] = ['range_plot','range_hist','range_mass']
  captions[section] = """ Inspiral Horizon distance for a \
       (1.4,1.4) solar mass system with SNR=8 (first plot), and \
       histograms(second plot). The last plot shows the \
       expected horizon distance for any total mass, using an SNR=8."""
  comments[section] = None
  configs[section] = None
  images_dirs[section] = images_dir

  section = 'numtemplates'
  sections.append(section)
  allIfosFlag.append(section)
  titles[section] = "Variation in template bank size"
  tags[section] = '*plotnumtemplates*'
  imagetags[section] = None
  captions[section] = "Variation in template bank size"
  comments[section] = None
  configs[section] = 'tmpltbank'
  images_dirs[section] = images_dir

  section = 'inspiral1'
  sections.append(section)
  titles[section] = "Pre-Coincident triggers"
  tags[section] = '*plotinspiral_FIRST_*'
  imagetags[section] = ['snr_vs_time','snr_cum_hist','snr_histogram']
  captions[section] = "Trigger rate at first inspiral stage"
  comments[section] = None
  configs[section] = 'inspiral'
  images_dirs[section] = images_dir

  section = 'ifar2'
  sections.append(section)
  titles[section] = "IFAR with CAT2 vetoes applied (including CBC hardware injections)."
  tags[section] = '*plotifar*'+ifartag[0]+'*CAT_2*'+ifartag[1]+'*'
  imagetags[section] = ['cumhist_uncombined_ifar','cumhist_combined_ifar']
  captions[section] = "Combined and uncombined IFAR vs cumulative number of triggers"
  comments[section] = None
  configs[section] = 'plotifar'
  images_dirs[section] = PDdir

  section = 'ifar3'
  sections.append(section)
  titles[section] = "IFAR with CAT2 vetoes applied and CBC hardware injections removed." 
  tags[section] = '*plotifar*'+ifartag[0]+'*CAT_3*'+ifartag[1]+'*'
  imagetags[section] = ['cumhist_uncombined_ifar','cumhist_combined_ifar']
  captions[section] = "Combined and uncombined IFAR vs cumulative number of triggers"
  comments[section] = None
  configs[section] = 'plotifar'
  images_dirs[section] = PDdir

  section = 'ifar4'
  sections.append(section)
  titles[section] = "IFAR with CAT2 and CAT3 vetoes applied and CBC hardware injections removed." 
  tags[section] = '*plotifar*'+ifartag[0]+'*CAT_4*'+ifartag[1]+'*'
  imagetags[section] = ['cumhist_uncombined_ifar','cumhist_combined_ifar']
  captions[section] = "Combined and uncombined IFAR vs cumulative number of triggers"
  comments[section] = None
  configs[section] = 'plotifar'
  images_dirs[section] = PDdir

  section = 'plotcumhist2'
  sections.append(section)
  titles[section] = "Effective SNR with CAT2 vetoes applied (including CBC hardware injections)."
  tags[section] = '*plotcumhist*'+ifartag[0]+'*CAT_2*'+ifartag[2]+'*'
  imagetags[section] = ['cumhist']
  captions[section] = 'Effective SNR vs number of triggers'
  comments[section] = None
  configs[section] = None
  images_dirs[section] = PDdir

  section = 'plotcumhist3'
  sections.append(section)
  titles[section] = "Effective SNR with CAT2 vetoes applied and CBC hardware injections removed."
  tags[section] = '*plotcumhist*'+ifartag[0]+'*CAT_3*'+ifartag[2]+'*'
  imagetags[section] = ['cumhist']
  captions[section] = 'Effective SNR vs number of triggers'
  comments[section] = None
  configs[section] = None
  images_dirs[section] = PDdir

  section = 'plotcumhist4'
  sections.append(section)
  titles[section] = "Effective SNR with CAT2 and CAT3 vetoes applied and CBC hardware injections removed."
  tags[section] = '*plotcumhist*'+ifartag[0]+'*CAT_4*'+ifartag[2]+'*'
  imagetags[section] = ['cumhist']
  captions[section] = 'Effective SNR vs number of triggers'
  comments[section] = None
  configs[section] = None
  images_dirs[section] = PDdir

  section = 'plotslides'
  sections.append(section)
  titles[section] = "Trigger rates at second coincidence stage (CAT2 and CAT3 vetoes applied and CBC hardware injections removed)"
  tags[section] = '*plotslides*'+ifartag[0]+'*CAT_4*'+ifartag[2]+'*'
  imagetags[section] = ['rates','durations_per_slide']
  captions[section] = 'Trigger rate at second coincidence stage'
  comments[section] = None
  configs[section] = None
  images_dirs[section] = PDdir

  if thisSearch=='full_data':
    section = 'zerofar2'
    sections.append(section)
    titles[section] = "FAR vs SNR for loud events with CAT2 vetoes applied" 
    tags[section] = '*lalapps_cbc_plotrates*CAT_2*'
    imagetags[section] = ['cumulative_F1']
    captions[section] = "SNR vs cumulative rate of triggers"
    comments[section] = None
    configs[section] = 'zerofar'
    images_dirs[section] = PDdir

    section = 'zerofar3'
    sections.append(section)
    titles[section] = "FAR vs SNR for loud events with CAT2 vetoes applied and CBC hardware injections removed." 
    tags[section] = '*lalapps_cbc_plotrates*CAT_3*'
    imagetags[section] = ['cumulative_F1']
    captions[section] = "SNR vs cumulative rate of triggers"
    comments[section] = None
    configs[section] = 'zerofar'
    images_dirs[section] = PDdir

    section = 'zerofar4'
    sections.append(section)
    titles[section] = "FAR vs SNR for loud events with CAT2 and CAT3 vetoes applied and CBC hardware injections removed." 
    tags[section] = '*lalapps_cbc_plotrates*CAT_4*'
    imagetags[section] = ['cumulative_F1']
    captions[section] = " vs cumulative number of triggers"
    comments[section] = None
    configs[section] = 'zerofar'
    images_dirs[section] = PDdir

  section = 'printlc2zl'
  sections.append(section)
  htmlOnly.append(section)
  titles[section] = "Loudest zero lag events with CAT2 vetoes applied (including CBC hardware injections)."
  htmltags[section] = '*'+ifartag[0]+'*CAT_2_VETO_LOUDEST*'+ifartag[1]+'*_EVENTS_BY_COMBINED_FAR*'
  captions[section] = 'Loudest events after Category 2 vetoes (including hardware injections).'
  comments[section] = None
  configs[section] = None
  images_dirs[section] = PDdir

  section = 'printlc3zl'
  sections.append(section)
  htmlOnly.append(section)
  titles[section] = "Loudest zero lag events with CAT2 vetoes applied and CBC hardware injections removed."
  htmltags[section] = '*'+ifartag[0]+'*CAT_3_VETO_LOUDEST*'+ifartag[1]+'*_EVENTS_BY_COMBINED_FAR*'
  captions[section] = 'Loudest events after Category 3 vetoes (including hardware injections).'
  comments[section] = None
  configs[section] = None
  images_dirs[section] = PDdir

  section = 'printlc4zl'
  sections.append(section)
  htmlOnly.append(section)
  titles[section] = "Loudest zero lag events with CAT2 and CAT3 vetoes applied and CBC hardware injections removed."
  htmltags[section] = '*'+ifartag[0]+'*CAT_4_VETO_LOUDEST*'+ifartag[1]+'*_EVENTS_BY_COMBINED_FAR*'
  captions[section] = 'Loudest events after Category 4 vetoes (hardware injections removed).'
  comments[section] = None
  configs[section] = None
  images_dirs[section] = PDdir

  section = 'printlcsl2'
  sections.append(section)
  htmlOnly.append(section)
  titles[section] = "Loudest time slide events with CAT2 vetoes applied (including CBC hardware injections)."
  htmltags[section] = '*'+ifartag[0]+'*CAT_2_VETO_LOUDEST_SLIDE_EVENTS_BY_COMBINED_FAR*'
  captions[section] = 'Loudest time slide events after Category 2 vetoes.'
  comments[section] = None
  configs[section] = None
  images_dirs[section] = PDdir

  section = 'printlcsl3'
  sections.append(section)
  htmlOnly.append(section)
  titles[section] = "Loudest time slide events with CAT2 vetoes applied and CBC hardware injections removed."
  htmltags[section] = '*'+ifartag[0]+'*CAT_3_VETO_LOUDEST_SLIDE_EVENTS_BY_COMBINED_FAR*'
  captions[section] = 'Loudest time slide events after Category 3 vetoes.'
  comments[section] = None
  configs[section] = None
  images_dirs[section] = PDdir


  section = 'printlcsl4'
  sections.append(section)
  htmlOnly.append(section)
  titles[section] = "Loudest time slide events with CAT2 and CAT3 vetoes applied and CBC hardware injections removed."
  htmltags[section] = '*'+ifartag[0]+'*CAT_4_VETO_LOUDEST_SLIDE_EVENTS_BY_COMBINED_FAR*'
  captions[section] = 'Loudest time slide events after Category 4 vetoes.'
  comments[section] = None
  configs[section] = None
  images_dirs[section] = PDdir

  page = create_sections(page,opts,sections,tags,captions,imagetags,\
                    images_dirs,titles,comments,configs,htmltags,htmlOnly,\
                    htmlAndPlot,allIfosFlag)

  return page


# ***************************************************************************
def write_injection(page, opts,injection, coinc_plots='coinc_result_plots'):
  """
  Creates the injection section

  @param page: the HTML document page. (type: markup.py document)
  @param opts: the main options variable (type: write_ihope_page options)
  @param injection: UNDOCUMENTED
  @param coinc_plots: UNDOCUMENTED
  @return: an update of HTML document
  """
  webdir = opts.webdir
  ini = opts.config_file
  ifos = get_ifos()
  images_dir = injection + '_summary_plots/'
  if opts.symlink_plots:
    symlink_plot_directory(opts.datadir + images_dir,\
                      opts.physdir + images_dir)
  else:
    copy_plot_directory(opts.datadir + images_dir,\
                      opts.physdir + images_dir)

  PDdir = coinc_plots + '/'
  if opts.symlink_plots:
    symlink_plot_directory(opts.datadir + PDdir,\
        opts.physdir + PDdir)
  else:
    copy_plot_directory(opts.datadir + PDdir,\
        opts.physdir + PDdir,open_box = opts.full_data)

  page.add("This section summarizes the analysis of the injection runs.<br/>")


  
  #---------------------------- the injection section
  # title will be the name of the section.
  # tags is a tag to search for the relevant cache file (will use the first one found)
  # imagetags. if not none, will only pu a subset of images corresponding to the list provided. 
  # captions is the text to be written in the caption
  # configs : is not none, the part of the ini file corresponding to this config name will be written in the web page
 
  sections = []
  htmlOnly = []
  htmlAndPlot = []
  allIfosFlag = []
  titles = {}
  tags = {}
  htmltags = {}
  imagetags = {}
  captions = {}
  comments = {}
  configs = {}
  images_dirs = {}

  section = 'pipedownfm2'
  sections.append(section)
  titles[section] = "Found and missed injections: Chirp mass vs decisive distance (CAT2 and CAT3 vetoes applied and CBC hardware injections removed)"
  tags[section] = '*ligolw_cbc_plotfm_fm_dist_v_param*'+injection.upper()+'_CAT_4_VETO*'
  imagetags[section] = ['ligolw_cbc_plotfm_fm_dist_v_param']
  captions[section] = 'Found and missed injections: injected decisive distance vs. injected mass. Stars are injections found with zero combined far; circles are injections found with non-zero combined far. Red crosses are missed injections (vetoed injections excluded). The decisive distance is the second smallest effective distance.'
  comments[section] = None
  configs[section] = None
  images_dirs[section] = PDdir

  section = 'pipedownfm1'
  sections.append(section)
  titles[section] = "Found and missed injections: End time vs decisive distance (CAT2 and CAT3 vetoes applied and CBC hardware injections removed)"
  tags[section] = '*ligolw_cbc_plotfm_fm_dist_v_end_time*'+injection.upper()+'_CAT_4_VETO*'
  imagetags[section] = ['ligolw_cbc_plotfm_fm_dist_v_end_time']
  captions[section] = 'Found and missed injections: injected decisive distance vs. injected end time by mass bin. Stars are injections found with zero combined far; circles are injections found with non-zero combined far. Red crosses are missed injections (vetoed injections are excluded). The decisive distance is the second smallest effective distance.'
  comments[section] = None
  configs[section] = None
  images_dirs[section] = PDdir

  section = 'pipedownacc1'
  sections.append(section)
  titles[section] = "Injection recovery: End time accuracy (CAT2 and CAT3 vetoes applied and CBC hardware injections removed)"
  tags[section] = '*ligolw_cbc_plotfm_fm_dist_v_dt*'+injection.upper()+'_CAT_4_VETO*'
  imagetags[section] = ['ligolw_cbc_plotfm_fm_dist_v_dt']
  captions[section] = 'Injected decisive distance vs difference between recovered and injected end times of "found" injections. Stars are injections found with zero combined far; circles are injections found with non-zero combined far. The decisive distance is the second smallest effective distance.'
  comments[section] = None
  configs[section] = None
  images_dirs[section] = PDdir

  section = 'pipedownacc2'
  sections.append(section)
  titles[section] = "Injection recovery: Chirp mass accuracy (CAT2 and CAT3 vetoes applied and CBC hardware injections removed)"
  tags[section] = '*ligolw_cbc_plotfm_fm_lin*'+injection.upper()+'_CAT_4_VETO*'
  imagetags[section] = ['ligolw_cbc_plotfm_fm_lin']
  captions[section] = 'Recovered parameter accuracies vs. difference between recovered and injected end times and injection tags. Stars are triggers with zero combined far, circles are triggers with non-zero far.'
  comments[section] = None
  configs[section] = None
  images_dirs[section] = PDdir

  section = 'chisq'
  sections.append(section)
  titles[section] = "The chisq parameter (CAT2 and CAT3 vetoes applied and CBC hardware injections removed)"
  tags[section] = '*plotsnrchi*CAT_4*'
  imagetags[section] = ['chisq_inj_newsnr_lines','rsq']
  captions[section] = "chi-square versus SNR and rsq versus SNR"
  comments[section] = "<span style=\"color:red\">This section is for the chisq vs snr plots. Unfortunately these are currently unavailable in pycbc. SOMEONE NEEDS TO FIX THIS!</span>"
  configs[section] = 'plotsnrchi'
  images_dirs[section] = images_dir

  section = 'ethinca'
  sections.append(section)
  titles[section] = "The Ethinca Parameter (CAT2 and CAT3 vetoes applied and CBC hardware injections removed)"
  tags[section] = '*plotethinca_SECOND*CAT_4*'
  imagetags[section] = ['ethinca_vs_combined']
  captions[section] = "Ethinca versus combined SNR in the different ifo combinations"
  comments[section] = "<span style=\"color:red\">This section is for the ethinca distribution plots. Unfortunately these are currently unavailable in pycbc. SOMEONE NEEDS TO FIX THIS!</span>"
  configs[section] = 'plotethinca'
  images_dirs[section] = images_dir

  section = 'injsqf'
  sections.append(section)
  htmlOnly.append(section)
  titles[section] = "Found injections that are not louder than the loudest slide (CAT2 and CAT3 vetoes applied and CBC hardware injections removed)"
  htmltags[section] = '*'+injection.upper()+'_CAT_4_VETO_QUIETEST_FOUND_COMPARED_TO_PLAYGROUND*'
  captions[section] = 'This table lists found injections that are not louder than the loudest event in the (full data) time slides. Equivalently, found injections that have a FAR greater than zero. These tables are shown after category 4 vetoes are applied.'
  comments[section] = None
  configs[section] = None
  images_dirs[section] = PDdir

  section = 'injscm'
  sections.append(section)
  htmlOnly.append(section)
  titles[section] = "Nearby missed injections (CAT2 and CAT3 vetoes applied and CBC hardware injections removed)"
  htmltags[section] = '*'+injection.upper()+'_CAT_4_VETO_CLOSEST_MISSED_INJECTIONS_SUMMARY*'
  captions[section] = 'This table shows a list of the closest injections that were not marked as "found" by our pipeline. These tables are shown after category 4 vetoes are applied.'
  comments[section] = None
  configs[section] = None
  images_dirs[section] = PDdir

  page = create_sections(page,opts,sections,tags,captions,imagetags,\
                    images_dirs,titles,comments,configs,htmltags,htmlOnly,\
                    htmlAndPlot,allIfosFlag)

  return page

# ***************************************************************************

def write_hw_injection(page, opts):
  """
  Creates the injection section

  @param page: the HTML document page. (type: markup.py document)
  @param opts: the main options variable (type: write_ihope_page options)
  @return: an update of HTML document
  """
  webdir = opts.webdir
  copy_plot_directory(opts.datadir + "hardware_injection_summary",\
      opts.physdir + "hardware_inj")  

  page.add("This section summarizes the analysis of the hardware injections.<br/>")
  
  sections = []
  htmlOnly = []
  htmlAndPlot = []
  allIfosFlag = []
  titles = {}
  tags = {}
  htmltags = {}
  imagetags = {}
  captions = {}
  comments = {}
  configs = {}
  images_dirs = {}

  section = 'hwinjpage1'
  sections.append(section)
  htmlAndPlot.append(section)
  titles[section] = "Hardware injection summary page at CAT 1"
  tags[section] = '*HWINJ_SUMMARY-*'
  htmltags[section] = '*HWINJ_SUMMARY-*'
  imagetags[section] = None
  captions[section] = ''
  comments[section] = None
  configs[section] = None
  images_dirs[section] = "hardware_inj"

  section = 'hwinjpage2'
  sections.append(section)
  htmlAndPlot.append(section)
  titles[section] = "Hardware injection summary page at CAT 2"
  tags[section] = '*HWINJ_SUMMARY_CAT_2*'
  htmltags[section] = '*HWINJ_SUMMARY_CAT_2*'
  imagetags[section] = None
  captions[section] = ''
  comments[section] = None
  configs[section] = None
  images_dirs[section] = "hardware_inj"

  page = create_sections(page,opts,sections,tags,captions,imagetags,\
                    images_dirs,titles,comments,configs,htmltags,htmlOnly,\
                    htmlAndPlot,allIfosFlag)

  return page

# ***************************************************************************
def write_about(page, opts):
  """
  Creates the section "About". 

  @param page: the HTML document page. (type: markup.py document)
  @param opts: the main options variable (type: write_ihope_page options)
  @return: an update of HTML document
  """
  webdir = opts.webdir
  
  page.p("This page was automatically generated with write_ihope_page using\
      the following ini file")
  tmp  = open(opts.config)
  tmp2 = tmp.read()
  tmp.close() 
  # the < character in a <pre> HTML tag is read and interpreted, So, we need to 
  # change it to &lt
  page.pre(tmp2.replace('<', '&lt;'))
  # page.pre(tmp.read())

  page.p("and the following command line arguments:")
  text=""
  for arg in sys.argv[:]:
    text = text +  arg +" "
  page.pre( text )
  
  page.p(__Id__[4:len(__Id__)-5])
  return page


# ***************************************************************************
def add_config_section(page, section):
  """
  Copy and paste a section of the ihope.ini into the HTML page within verbatim
  tags

  @param page: the HTML document page. (type: markup.py document)
  @param section: the name of the section to copy and paste (type: string)
  @return: an update of HTML document
  """
  # section may be empty or set to None
  try:
    ini  = hipecp.items(section)
  except:
    return page

  page.add("<pre>")
  page.add("------------------------------------" +section)
  for i in  ini :
    page.add(i[0]+' = '+i[1])
  page.add("</pre>")
  return page


# ***************************************************************************
def heading(page, title="None", label="Switch details on/off", heading="h3"):
  """
  Add a hx HTML section within the document with the ability to toggle it on/off
   
  @param page: the HTML document page. (type: markup.py document)
  @param title: the name of the section (type: string)
  @param label: the label to put on the toggle button  (type: string)
  @param heading: the HTML heading (h3 by default) (type: string)
  @return: an update of HTML document
  """
  #increment block number
  global count_block
  global h2_num,h3_num
  global fig_num

  input=str(count_block)
  count_block=count_block+1

  fig_num = 1

  if heading == 'h2':
    section_num = str(h2_num)+"."
  elif heading == 'h3':
    section_num = str(h2_num)+"."+str(h3_num)+"."
  else:
    raise ValueError, "heading must be either h2 or h3 in heading function"


  page.add("<"+heading+">"+ section_num+ title)
  h3_num = h3_num + 1
  text = label 

  page.add("</"+heading+">")
  page.div(id="div_"+input , style='display:none') 

  logText(logfile, '      Enters sub-section: '+title )

  return page 

# ***************************************************************************
def add_caption(page, caption):
  """
  Add a caption to the HTML document. Should be used with the figure only. 
  Take care of the figure numbering.

  @param page: the HTML document page. (type: markup.py document)
  @param caption: the name of the section to copy and paste (type: string)
  @return: an update of HTML document
  """
  global fig_num
  page.p("<b>Figure "+str(fig_num) + ":</br> "+caption)
  fig_num = fig_num + 1

  return page


# ***************************************************************************
def add_figure(page,webDir,fnames="test", caption="add a caption", size=None, \
    alt="no figure found",source="not specified",html=None,html_file = None, \
    all_ifos = False):
  """
  Add a figure to the HTML document, taking care of the numbering.
 
  @param page: the HTML document page. (type: markup.py document)
  @param webDir: UNDOCUMENTED
  @param fnames: UNDOCUMENTED
  @param caption: the name of the section to copy and paste (type: string)
  @param size: the size of the image ("full", "half", "third") (type: string)
  @param alt: an HTML alt  (type: a string)
  @param source: the source of the figure (type: string)
  @param html: UNDOCUMENTED
  @param html_file: UNDOCUMENTED
  @param all_ifos: UNDOCUMENTED
  @return: an update of HTML document
  """
  global fig_num
  page.add("<!-- insert a figure -->\n<div class=\"figure\">")
  this_count = 0
 
  # set the size of the images
  if size==None:
    if len(fnames)==1: 
      size="full"
    elif len(fnames)==2:
      size="half"
    else : size="third"

  # Determine the ifo
  ifo = None
  if html:
    for combo in ['H1','H2','L1','V1','H1L1','H1H2','H2L1','H2V1','H1V1','L1V1','H1L1V1','H1H2L1','H2L1V1','H1H2V1','H1H2L1V1']:
      if html.startswith(combo):
        ifo = combo
  if ifo and not all_ifos:
    page.h5(ifo + ' were operating')

# Source is the thumnail, target is the full-resolution image. 

  for fnam in fnames:
    source=webDir+"/"+fnam.strip()
    title  = ' title="%s" ' % alt[this_count].strip()
    title  = title.replace("_thumb","")
    target = source.replace("_thumb","")
    if size == 'full':
      source = source.replace("_thumb","")
    page.add("\t<a href=\"" + target+"\"" +  title + " rel=\"external\">\n" )
    try:
      thisalt = alt[this_count]
      thisalt.replace("\n","")
      page.img(class_=size ,src=source, alt=thisalt )
    except:
      page.img(class_=size ,src=source )
      
    page.add("</a>")
    this_count = this_count +1

  section_num = str(h2_num)+"."+str(h3_num-1)+"."+str(fig_num)
  page.add("\t<p><b>Figure " + section_num + "</b>:  " + caption + "</p>")
  fig_num +=1

  # add a yellow box (depends on yourcss style though) that contains the 
  # filename of the pictures.
  if fnames is not None:
    page.add("<pre>Source: "+webDir+"\n" )
    for fnam in fnames:
      page.add(fnam.replace("\n",""))
    page.add("</pre>")

  if html_file:
    this = open(html_file, "r")
    for line in this:
      page.add(line)
    page.add("<pre>Source: "+ html_file + "\n </pre>" )
    this.close()

  page.div.close()

  #check that the file exsits. If not, returns an error message
  return page

# ***************************************************************************
def create_venn(data, tag):
  """
  Create a venn diagram for the 2 or 3 ifos case
  data has to be H1H2 H1 H2L1 H2 H1L1 L1 H1H2L1 array 

    >>> data = 
    >>> tag = 
    >>> create_venn(data, tag)

  @param data: an input dictionary
  @param data: numbers
  @param tag:  (type: string)
  @return: a boolean. True is succesfule, False otherwise.
  """

  try:
    mscript = open("temp.m", "w")
    mscript.write("data = [")
    mscript.write( data +"];" ) 
    mscript.write(" vennX(data\'/3600/24,0.01);")
    mscript.write(" k=colormap(jet); k = [1 1 1; k];colormap(k);")
    mscript.write("saveas(gcf,\'venn_"+tag+".png\')")
    mscript.close()
    command=("matlab -nodisplay -nodesktop -nojvm -nosplash   < temp.m > /tmp/test ;  rm -f temp.m; mv venn_"+tag+".png "+opts.physdir+"/"+tag+"/")
    if not opts.debug:
      make_external_call(command, opts.debug, opts.debug, True)  
    return True 
  except:
    logText(logfile, """WARNING   The matlab command to create the venn diagram failed. 
                Check that matlab is properly set,and vennX.m is available
                (see matapps/src/searches/inspiral/matspiral/utilities")  
          """, "error")
    return False

# ***************************************************************************
def get_coincident_segments(tag):
  """
  @param tag: "playground" or "full_data"  (type: string)
  return: a dictionary with the coincident time for each ifo combination
  """
  ifos = get_ifos()
  thisdata = {}
  thisdata['segments'] = ['duration(days)' ,'duration (s)']
  output={}
  ifo_coincs = get_ifo_coinc()  

  try:
    for coinc in ifo_coincs:
      if tag=="playground":
        command = "awk \'{sum=sum+$4} END {print sum}\' "\
          +opts.datadir+ tag +"/"+coinc+"_play_segs_analyzed.txt"
      elif tag=="full_data" or tag=="full_data_slide":
        command = "awk \'{sum=sum+$4} END {print sum}\' "\
          +opts.datadir+"full_data/"+coinc+"_segs_analyzed.txt"

      output[coinc], status = \
        make_external_call(command, False,opts.debug, True)

    logText(logfile, '...Get the analyzed segments duration...')
  except:
    logText(logfile , 'problem(s) while parsnig the coincident segments', \
      "error")

  return output

  


# ***************************************************************************

def get_segs_from_file(filename, segment_name, version=None):
  """
  Read a segment.xml file and return a dictionary of segment list matching
  segment_name. If version is supplied it will also match that. The provided
  segment_name should *not* contain the ifo or the version.
  """
  xmldoc = utils.load_filename(filename, contenthandler=ContentHandler)
  segs = segment_utils.LigolwSegments(xmldoc)
  segs_dict = {}
  for seg in segs:
    if seg.name == segment_name:
      if len(seg.instruments) != 1:
        raise ValueError("Expected exactly one ifo for each segment_id")
      ifo = ''.join(seg.instruments)
      if segs_dict.has_key(ifo):
        raise ValueError("Expected each ifo to only appear at one segment_id")
      segs_dict[ifo] = seg
  return segs_dict


# ***************************************************************************
def get_ifo_coinc():
  """
  return: list of valid coincidences with respect to the ini file

  """
  # FIXME: This function is horrible. It probably also exists, nicer, somewhere
  #        else.
  # get the ifo requested
  ifos = get_ifos()
  # get number of ifos to look at
  numifos = []  
  for option in ["two-ifo","three-ifo","four-ifo", "five-ifo"]:
    tmp = option.split("-")
    numifos.append(tmp[0])
  # now let us fill the different possible ifo combinations
  output = []
  for num in numifos:
    if num=="one":
      for ifo in ifos:
        output.append(ifo)
    elif num=="two":
      for ifo1 in ifos:
        for ifo2 in ifos:
          if ifo1 < ifo2:
            output.append(ifo1+ifo2)
    elif num=="three":
      for ifo1 in ifos:
        for ifo2 in ifos:
          for ifo3 in ifos:
            if ifo1 < ifo2 and ifo2 < ifo3:
              output.append(ifo1+ifo2+ifo3)
    elif num=="four":
      for ifo1 in ifos:
        for ifo2 in ifos:
          for ifo3 in ifos:
            for ifo4 in ifos:
              if ifo1 < ifo2 and ifo2 < ifo3 and ifo3 < ifo4:
                output.append(ifo1+ifo2+ifo3+ifo4)
  return output


  


# ***************************************************************************
def get_ifos():
  """
  read the ifos used in the ifo-details section of the pycbc ini file
  return: list of ifos
  """
  ifos=[]
  for ifo in hipecp.options('workflow-ifos'):
    ifos.append(ifo.upper())

  return ifos

# ***************************************************************************
def copy_segments():
  """
  This function copies the segments list into the web page directory 
  """

  msg =   "Copying segments into the web page directory (in yoururl/segments)"
  logText(logfile, msg)
  # first we create this directory      
  mkdir(opts.physdir+'/segments')
  mkdir(opts.physdir+'/catlists')
  # parsing the ini file, find the cat file and thenread the ./segments directory
  try:
    location = opts.physdir + '/segments/'
    if opts.no_copy is False:
      command = 'cp '+opts.datadir +'/segments/* ' + location
      dummy,status = make_external_call(command, opts.debug, opts.debug, True)
      command = 'cp '+opts.datadir +'/segments/*cat*.txt ' + opts.physdir + '/catlists/' 
      dummy,status = make_external_call(command, opts.debug, opts.debug, True)
  except:
    logText(logfile, "Could not copy the segment files", "error")
    pass

 #  the selected segment files
  for thisSearch in ['playground', 'full_data']:
    mkdir(opts.physdir+thisSearch)
    try :
      stdout ='Copying the selected segments files into /'+thisSearch
      for this in get_ifo_coinc():
          if thisSearch=='playground':
            seg = '/'+thisSearch+'/'+this +'_play_segs_analyzed.txt'
          elif thisSearch=='full_data':
            seg = '/'+thisSearch+'/'+this +'_segs_analyzed.txt'

          if opts.no_copy is False:
            dest = opts.physdir + thisSearch + '/segments'
            mkdir(dest)
            command = 'cp '+opts.datadir + seg + ' ' + dest
            dummy,status = \
                make_external_call(command, opts.debug, opts.debug, True)

          if status>0:
            stdout += " WARNING: could not copy a selected segment file "
            stdout += "from " + thisSearch + " for " + this + "ifo combination"
    except:
      stdout +=" WARNING: problem while copying a selected segment"
      stdout += "(from " + thisSearch + "). passing..."    
      logText(logfile,  stdout, "warning")
      pass
    else: 
      logText(logfile,  stdout)

def copy_files(dataDir,webDir,name):
  '''
  A function to use python to copy files matching name from dataDir to webDir
  '''
  # Loop over matching files
  for file in glob.glob(os.path.join(dataDir,name)):
    # Copy the file
    shutil.copy(file, webDir)

def copy_plot_directory(dataDir,webDir,open_box = True):
  '''
  A function to copy the XXX_summary_plots directory to the html directory
  '''
  # The open_box flag is an old flag that was used to not copy open box plots
  # At the moment this flag is unused but is kept as it may be used again.
  if not os.path.isdir(webDir):
    mkdir(webDir)
    print >> sys.stdout, ' .../ copying all' + dataDir + ' files to ' + webDir
    # Copy the html and cache files
    copy_files(dataDir,webDir,'*html')
    copy_files(dataDir,webDir,'*cache')

    # Copy the images directory
    from_path = os.path.join(dataDir, 'Images')
    if os.path.isdir(from_path):
      to_path = os.path.join(webDir, 'Images')
      if os.path.exists(to_path):
        shutil.rmtree(to_path)
      shutil.copytree(from_path, to_path)
    else:
      # Why does the directory not exist?
      errMsg = "Directory %s does not exist." %(from_path,)
      logText(logfile, errMsg, "warning")

    # Copy the omega_scans directory
    from_path = os.path.join(dataDir, 'omega_scans')
    if os.path.isdir(from_path):
      to_path = os.path.join(webDir, 'omega_scans')
      if os.path.exists(to_path):
        shutil.rmtree(to_path)
      shutil.copytree(from_path, to_path)
    else:
      # Why does the directory not exist?
      errMsg = "Directory %s does not exist." %(from_path,)
      logText(logfile, errMsg, "warning")

    print >> sys.stdout, '... Done'

def symlink_plot_directory(dataDir,webDir):
  '''
  A function to symlink, rather than copy, the plot directory to the html
  directory
  '''
  if not os.path.isdir(webDir):
    if dataDir[-1] == '/':
      dataDir = dataDir[:-1]
    if webDir[-1] == '/':
      webDir = webDir[:-1]
    command = 'ln -s ' + dataDir + ' ' + webDir
    print command
    print >> sys.stdout, ' .../symlinking all' + dataDir + ' files to ' + webDir
    make_external_call(command, opts.debug, opts.debug, True)
    print >> sys.stdout, '... Done'

    
# ***************************************************************************
def fom(page, opts, cachefile_tag=None, caption="fix me",\
	 image_tag=None,directory="playground_summary_plots",all_ifos=False):
  """
  This function reads a cachefile, copy the files to the relevant directory, and  update the HTML document to add figures and pertinent sections. 

  
  @param page: the HTML document page. (type: markup.py document)
  @param opts: the main options variable (type: write_ihope_page options)
  @param cachefile_tag: the pattern of the cachefile to look at (type: string)
  @param caption: a list of caption (type: list of string )
  @param image_tag: a list of tag (including *) to select files within a
  @param directory:  the directory to look at (type: string)
  @param all_ifos: UNDOCUMENTED

  cachefile
  """

  dataDir = opts.datadir+directory+'/'
  webDir = opts.physdir + directory + '/'

  # create a list of cachefiles, each of which containing
  # the cachefile_tag argument in its name
  thisglob = webDir + cachefile_tag +'cache'
  cachefileList =  glob.glob(thisglob)
  
  if opts.verbose is True: 
    print "        Searching for files with this(ese) tag(s): " +str(image_tag)

  # for each cachefile we create a div figure section with all 
  # figures whose name is part of the image_tag list
  cachefileList.sort()
  for eachcache in cachefileList:
    # read this cachefile
    this = open(eachcache, "r")
    fnameList = []
    # Lots of lists but it ensures that the order is always the same
    fileNameList = []
    href = None
    #for each file contained in the cachefile
    if opts.debug is True :
      print >>sys.stdout, "        --> Copying files from " +eachcache

    for filename in this:
      filename = filename.replace("\n","")
      filename = filename.replace(" ","")
      fileNameList.append(filename)
    
    fileNameList.sort()
 
    for filename in fileNameList:
      # if the file is an image, we copy it in ./Images
      # if the file is an image, we copy the file. Then, if the pattern 
      # (in image_tag)  matches the filename, then we create am html section.
      if filename[-4:]=='.png':
        # we check if the pattern match the file
        if image_tag is not None:
          for eachPattern  in image_tag:
            if patternFoundInFilename(filename, eachPattern) is True:
              fnameList.append(filename.replace(".png", "_thumb.png"))
        else:
          fnameList.append(filename.replace(".png", "_thumb.png"))
      elif filename[-5:] == '.html' and ('followup' not in filename):
        href = filename.replace("/pictures", "")
        href = href.replace("\n", "")
        htmlFile = href
        href = directory + '/' + href

    if href:
      msg =" <a href=\"" + href + "\""
      msg+=" rel=\"external\" >"
      msg += """ <br/> --- <b>Click here (to open link in a new tab/window)</b> to get all pictures (full resolution) 
          as well as the pylal arguments used to generate the plots</a> """
      
      source =  eachcache.split('/') 
      source =  source[len(source)-1]
      page = add_figure(page, directory, fnames=fnameList, \
          caption=(caption+' '+msg), source=source,size=None, alt=fnameList,\
          html = htmlFile,all_ifos=all_ifos)
      # close the cachefile
    this.close()
  
#  page.div.close()

  return page
# ***************************************************************************

def html_and_plot(page, opts, cachefile_tag=None,\
          caption='fix_me',image_tag=None,\
          directory=None,html_tag=None,all_ifos=False):
  """
  This function does fom and html_insert together!
  """
  dataDir = opts.datadir+directory+'/'
  webDir = opts.physdir + directory + '/'
  # create a list of cachefiles, each of which containing
  # the cachefile_tag argument in its name
  thisglob = webDir + cachefile_tag +'cache'
  cachefileList =  glob.glob(thisglob)
  # And do the same for the html_tag glob.
  thisglob = webDir + html_tag + 'html'
  htmlfileList = glob.glob(thisglob)

  # Now we need to match them up
  fileListDict = []
  cacheFiles = {}
  htmlFiles = {}
  cachefileList.sort()
  for file in cachefileList: 
    ifo = None
    for combo in ['H1','H2','L1','V1','H1L1','H1H2','H2L1','H2V1','H1V1','L1V1','H1L1V1','H1H2L1','H2L1V1','H1H2V1','H1H2L1V1']:
      if os.path.basename(file).startswith(combo):
        ifo = combo
    if ifo:
      cacheFiles[ifo] = file
      fileListDict.append(ifo)
    else:
      print >> sys.stderr,"WARNING: file doesn't seem to match an ifo combo!"

  for file in htmlfileList:
    ifo = None
    for combo in ['H1','H2','L1','V1','H1L1','H1H2','H2L1','H2V1','H1V1','L1V1','H1L1V1','H1H2L1','H2L1V1','H1H2V1','H1H2L1V1']:
      if os.path.basename(file).startswith(combo):
        ifo = combo
    if ifo:
      htmlFiles[ifo] = file
    else:
      print >> sys.stderr,"WARNING: file doesn't seem to match an ifo combo!"
    if ifo not in cacheFiles.keys():
      print >> sys.stderr,"WARNING: file doesn't seem to correspond to cache file list"

  if opts.verbose is True:
    print "        Searching for files with this(ese) tag(s): " +str(image_tag)

  for ifo in fileListDict:
    if ifo in htmlFiles.keys():
      # read the image cachefile
      this = open(cacheFiles[ifo], "r")
      fnameList = []
      # Lots of lists but it ensures that the order is always the same
      fileNameList = []
      href = None
      #for each file contained in the cachefile

      if opts.debug is True:
        print >>sys.stdout, "        --> Copying files from " +cacheFiles[ifo]

      for filename in this:
        filename = filename.replace("\n","")
        filename = filename.replace(" ","")
        fileNameList.append(filename)

      fileNameList.sort()

      for filename in fileNameList:
        # if the file is an image, we copy it in ./Images
        # if the file is an image, we copy the file. Then, if the pattern 
        # (in image_tag)  matches the filename, then we create am html section.
        if filename[-4:]=='.png':
          # we check if the pattern match the file
          if image_tag is not None:
            for eachPattern  in image_tag:
              if patternFoundInFilename(filename, eachPattern) is True:
                fnameList.append(filename.replace(".png", "_thumb.png"))
          else:
            fnameList.append(filename.replace(".png", "_thumb.png"))
        elif filename[-5:] == '.html' and ('followup' not in filename):
          href = filename.replace("/pictures", "")
          href = href.replace("\n", "")
          htmlFile = href
          href = directory + '/' + href

      if href:
        msg =" <a href=\"" + href + "\""
        msg+=" rel=\"external\" >"
        msg += """ <br/> --- <b>Click here (to open link in a new tab/window)</b> to get all pictures (full resolution) 
            as well as the pylal arguments used to generate the plots</a> """
  
        source =  cacheFiles[ifo].split('/')
        source =  source[len(source)-1]
        page = add_figure(page, directory, fnames=fnameList, \
            caption=(caption+' '+msg), source=source,size=None, alt=fnameList,\
            html = htmlFile,html_file = htmlFiles[ifo],all_ifos=all_ifos)
        # close the cachefile
        this.close()

  return page
  
###############################################################################

def html_insert(page,opts, html_tag=None,\
          caption='fix_me',directory='playground_summary_plots',all_ifos=False):
  """
  This function copies a set of html files verbatim into the page. 

  
  @param page: the HTML document page. (type: markup.py document)
  @param opts: the main options variable (type: write_ihope_page options)
  @param html_tag: the pattern of the html files to look at (type: string)
  @param caption: a list of caption (type: list of string )
  @param directory:  the directory to look at (type: string)
  @param all_ifos: UNDOCUMENTED
  """

  dataDir = opts.datadir+directory+'/'
  webDir = opts.physdir + directory + '/'

  # create a list of html files to be used
  thisglob = webDir + html_tag +'html'
  htmlfileList =  glob.glob(thisglob)

  if opts.verbose is True:
    print "        Copying this html file : " +str(html_tag)

  # for each cachefile we create a div figure section with all 
  # figures whose name is part of the image_tag list
  global fig_num
  htmlfileList.sort()
  for htmlfile in htmlfileList:
    # read this file
    this = open(htmlfile, "r")
    page.add("<!-- insert a figure -->\n<div class=\"figure\">")
    this_count = 0
   # Determine the ifo
    ifo = None
    for combo in ['H1','H2','L1','V1','H1L1','H1H2','H2L1','H2V1','H1V1','L1V1','H1L1V1','H1H2L1','H2L1V1','H1H2V1','H1H2L1V1']:
      if os.path.basename(htmlfile).startswith(combo):
        ifo = combo
    if ifo and not all_ifos:
      page.h5('When ' + ifo + ' interferometers operating')
    # Add the html file into the page
    for line in this:
      page.add(line)
    # add a yellow box (depends on yourcss style though) that contains the 
    # html filename.
    section_num = str(h2_num)+"."+str(h3_num-1)+"."+str(fig_num)
    page.add("\t<p><b>Figure " + section_num + "</b>:  " + caption + "</p>")
    fig_num +=1
    page.add("<pre>Source: "+ htmlfile + "\n </pre>" )
    page.div.close()
    # close the file
    this.close()

  return page

# ***************************************************************************
def set_style():
  """
  Function to copy the style file as read from the write_ihope_page.ini file
  """

  tmp = []
  tmp.append(configcp.get("main", "style"))
  try:
    style = configcp.get("main","style")
    command = 'cp ' + style + " " +opts.physdir
    make_external_call(command, opts.debug, opts.debug, True )
    tmp=style.split('/')
    return tmp[len(tmp)-1]
    
  except:
    print sys.stderr()<< 'could not copy the style file'
    pass
    return ""

# ----------------------------------------------------------------------------
def parse_arguments():
  """
  Function to parse the arguments and check their validity.
  """
  usage =  """ %prog [options]
  Program to write webpage from upperlimit.py
  """

  parser = OptionParser( usage = usage, version = "%prog CVS "+__Id__ )

  parser.add_option("-C","--config-file",action="store",type="string",\
      metavar=" INI File",\
      help="""ini file with information about run directories. The ini file should look like
-------                        
[main].............................................
gps-start-time  = 847555570........................
gps-end-time    = 849974770........................
title           = \"Low mass CBC analysis\"........
ihope-ini-file  = ihope.ini........................
ihope-directory = /archive/home/cokelaer/S5/Month1/full_analysis/....
home-directory  = /archive/home/...................
url             = ldas-jobs.ligo.caltech.edu.......
username        = cokelaer.........................
style           = /archive/home/cokelaer/style.css.
output          = index.shtml""")
  parser.add_option("-A","--open-the-box",action="store_true",\
      default=False,dest="full_data", metavar="DOANALYSIS",\
      help="" )
  parser.add_option("-T","--symlink-plots",action="store_true",\
      default=False, metavar="DOTUNING",\
      help="" )
  parser.add_option("-S","--skip-summary",action="store_false",\
      default=True,dest="summary", metavar="SOSUMMARY",\
      help="Skip generating brief summary pages")
  parser.add_option("-U","--skip-upperlimit",action="store_false",\
      default=True,dest="upperlimit", metavar="DOUPPERLIMIT",\
      help="" )
  parser.add_option("-I","--skip-injection",action="store_false",\
      default=True,dest="injection", metavar="DOUPPERLIMIT",\
      help="" )
  parser.add_option("-H","--skip-hardware-injection",action="store_false",\
      default=True,dest="hardware_injection", metavar="SKIPHWINJ",\
      help="" )
  parser.add_option("-P","--skip-playground",action="store_false",\
      default=True,dest="playground", metavar="DOPLAYGROUND",\
      help="" )
  parser.add_option("-B","--skip-full-data-slide",action="store_false",\
      default=True,dest="full_data_slide", metavar="DOPLAYGROUND",\
      help="" )
  parser.add_option("-N","--no-copy",action="store_true",\
       default=False,dest="no_copy",metavar="NOCOPY",\
       help="do not copy any file (useful for debugging only) ")
  parser.add_option("-D","--debug",action="store_true",\
       default=False,dest="debug",metavar="DODEBUG",\
       help="More verbose than --verbose ")
  parser.add_option("-V","--verbose",action="store_true",\
      default=False, dest="verbose",metavar="VERBOSE",\
      help="talkative script! One can also add the debug option." )

  (opts,args) = parser.parse_args()

  if opts.config_file is None:
    raise ValueError

  return opts,args

# ----------------------------------------------------------------------------

opts,args = parse_arguments()
#############################################################################
#  MAIN PART                                                                #
#############################################################################
fig_num = 1
count_block = 0  # counter for the unique id of the toggle boxes
h2_num = 0       # counter for the section h2
h3_num = 1       # counter for the section h3
config   =  opts.config_file
opts.config = config # save the name of the ini file, why ?
configcp = glue.pipeline.DeepCopyableConfigParser()
configcp.read(config)
maxdiv = 200
print >>sys.stdout, "|------------------- Initialisation"
# First, we open an xml file, for the log file
logfile_name = __name__+".xml"
print >>sys.stdout,"Openning the log file (" +logfile_name+")."
logfile = open(logfile_name, "w")
logfile.write("""<?xml version="1.0" encoding="ISO-8859-1"?>
<?xml-stylesheet type="text/xsl" href="write_ihope_page.xsl"?>
<log>""")


#---------------------------------------
# then, we parse the write_ihope_page.ini file
print >>sys.stdout,"Parsing the ini file: " + opts.config
try:
  opts.config_file 	= configcp.get("main", "ihope-ini-file")
  opts.gps_start_time 	= configcp.get("main", "gps-start-time")
  opts.gps_end_time 	= configcp.get("main", "gps-end-time")
  opts.ihope_directory 	= configcp.get("main", "ihope-directory")
  opts.title	 	= configcp.get("main", "title").replace( '"', '' )
  opts.output	 	= configcp.get("main", "output")
  opts.html_directory 	= configcp.get("main", "html-directory")
  Lead                  = configcp.get("main", "lead").replace( '"', '' )
  Second                = configcp.get("main", "second").replace( '"', '' )
  Notes                 = configcp.get("main", "notes").replace( '"', '' )
except:
  print >> sys.stderr, "ERROR::The ini file does not have the proper field in the [main] section" 
  print >> sys.stderr, """       Consider adding one of those fields if missing: ihope-ini-file, \
	gps-start-time,gps-end-time, ihope-directory, title,url, username, output"""
  raise
  
#------------------------------------
#sub-products of the ini file parsing
opts.gpsdir =  '/'+str(opts.gps_start_time)+'-'+str(opts.gps_end_time)+'/'   # note the / at the end
opts.duration = str(int(opts.gps_end_time) - int(opts.gps_start_time))
opts.daydur = float(opts.duration) / 86400
opts.webdir = './'
opts.datadir = os.path.abspath(opts.ihope_directory) + opts.gpsdir 
opts.physdir = opts.html_directory + opts.gpsdir
opts.txt_suffix = '-'+opts.gps_start_time+'-'+opts.duration+'.txt'


#----------------------
# openning the html file
print >>sys.stdout,"Openning the HTML (" + opts.output+")"
try:
  html_file = file(opts.output,"w")
except:
  msg = "Cannot open %" % opts.output
  print >>sys.stderr, msg
  raise


#-----------------------------------------
# here is the directory we want to extract information from
msg = "Entering this directory (where we will get all the relevant information)" + opts.datadir
print >> sys.stdout, msg
if not  os.path.isdir(opts.datadir):
  raise  "%s is not a valid directory. Check your gps time." % opts.datadir
# which physical name is 
msg = "We will copy all images and HTML documents into this directory " +opts.physdir
logText(logfile, msg,"done")
# Make the physdir and all necessary directories above it
try:
  os.makedirs(opts.physdir)
except OSError:
  pass
mkdir(opts.physdir+'/Images')

# get the style file and copy it (must happen after the html directories have been created)
opts.style = set_style()

#-----------------------------------------
# now we can parse the ihope.ini file itself
msg =   "Parsing the ihope ini file (" + opts.config_file+")"
logText(logfile, msg)
hipe   = opts.datadir+'/'+opts.config_file
hipecp = glue.pipeline.DeepCopyableConfigParser()
hipecp.read(hipe)
make_external_call( 'cp '+opts.config_file + ' ' + opts.physdir, False, opts.debug, True)

#-----------------------------------------
# now we copy the segments to the web directory and other relevant information
copy_segments()


#-----------------------------------------
###### create the section labels  
html_sections={}

# the string "full_data" is hardcoded somewhere else, so it must remain as it is
html_order = ['general', 'datainfo', 'fullsummary','play_full_slide_summary',\
    'playsummary','playground', 'full_data', \
    'full_data_slide', 'upperlimit_full','upperlimit_play','injection',\
    'hardware_injections', 'upperlimit', 'logfile', 'about']

html_sections['general'] = "General Information"
html_sections['datainfo'] = "Data Information"
defaultPage = 'datainfo'
if opts.summary is True and opts.playground is True:
  html_sections['playsummary'] = "Summary of playground"
  defaultPage = 'playsummary'
if opts.summary is True and opts.full_data_slide is True:
  html_sections['play_full_slide_summary'] = \
      "Summary of playground with full data slides"
  defaultPage='play_full_slide_summary'
if opts.summary is True and opts.full_data is True:
  html_sections['fullsummary'] = "Summary of full data"
  defaultPage = 'fullsummary'
if opts.playground is True: html_sections['playground']	= "Playground"
if opts.full_data_slide is True: 
  html_sections['full_data_slide'] = "Full Data Slide And Playground Zero lag"
if opts.injection is True: 
  if hipecp.has_section("injections"):
    html_sections['injection'] = "Injection"
    for inj in hipecp.options("injections"):
      html_sections[inj] = inj.upper() 
    html_sections['allinj'] = 'All Injections Combined'
  else:
    opts.injection = False
    print >> sys.stderr, "No injections section found in ini file."
    print >> sys.stderr, "Now running with --skip-injections"
if opts.hardware_injection is True: 
  html_sections['hardware_injections'] = "Hardware Injection"
if opts.full_data is True: html_sections['full_data'] = "Full Data"
if opts.upperlimit is True: 
  html_sections['upperlimit_play'] = "Search Sensitivity"
if opts.upperlimit and opts.full_data: 
  html_sections['upperlimit_full'] = "Upper Limits"
html_sections['logfile'] = "Log File"
html_sections['about'] = "About"

title = opts.title+ ": from "+str(opts.gps_start_time)\
  +" to "+str(opts.gps_end_time) 
script = {}
script['toggle.js'] = 'javascript'
script['//ajax.googleapis.com/ajax/libs/jquery/1.2.6/jquery.min.js']\
    = 'javascript'
create_toggle()
# Finally, we create the html document 
msg =   "Creating HTML document"
logText(logfile, msg)

mainpage = initialize_page(title,opts.style,script)

mainpage.div(id_="wrapper")
mainpage.div(id_="menubar")
mainpage.div(id_="menu")
# The main part of the code, which loops over all subsections
subPages = {}
for each_section in html_order:
  if each_section in html_sections:
    if each_section == 'injection':
      injs = hipecp.options("injections")
      injs.sort()
      for inj in injs:
        subPages[inj]=\
            write_sub_page(opts,inj,html_sections,opts.style,script)
      subPages[inj]=\
          write_sub_page(opts,'allinj',html_sections,opts.style,script)
    else:
      subPages[each_section]=\
          write_sub_page(opts,each_section,html_sections,opts.style,script)
mainpage.div.close()
mainpage.div.close()
mainpage.div(id_="ihope")
mainpage.add('<h2> i h o p e </h2>')
if opts.full_data:
  mainpage.add('<img src="https://www.lsc-group.phys.uwm.edu/ligovirgo/cbc/public/segments/S5/OpenBox.jpg">')
else:
  mainpage.add('<img src="https://www.lsc-group.phys.uwm.edu/ligovirgo/cbc/public/segments/S5/thomasLegacy.jpg">')
mainpage.div.close()
mainpage.div(id_='header')
mainpage.add('<h1>' + title  +' </h1>')
leads = 'Lead: ' + Lead + '&nbsp;&nbsp&nbsp Second: '+ Second
mainpage.add('<h3> ' + leads + ' </h3>')
mainpage.add('<h3>' + Notes + ' </h3>')
mainpage.div.close()
mainpage.div(id_='iframecontent')
mainpage.add('<p id="placeholder">Please select a report section on the left.</p>')
mainpage.add('<script type="text/javascript">')
mainpage.add("""loadFrame('""" + defaultPage + """.html')""")
mainpage.add('</script>')
mainpage.div.close()
mainpage.div.close()

# what is the False for ? I cannot remenber
html_file.write(mainpage(False))
html_file.close()
# just remove the <html lang="en"> line tat is not a strict HTML code
cmd = 'sed -e \'s/<html lang=\"en\">//\' '+ opts.output + '>/tmp/tmp.html'
output,status = make_external_call(cmd, True, True, False)
if status==0:
  cmd = 'mv /tmp/tmp.html '+opts.output
  make_external_call(cmd, True, True, True)
if opts.full_data:
  cmd = 'sed -i \'s/<body>/<body\ class=\"openbox\">/g\' ' + opts.output + '>/tmp/tmp.html'
  make_external_call(cmd, True, True, True)

# that's it for the html creation. let us copy it to the requested directory  

print '---------------------FINISHED ---------------------'
print '--- HTML file created. '
print '--- Copying html documents in ' +opts.physdir
make_external_call('mv  '+opts.output +' ' + opts.physdir, opts.debug, opts.debug, True)
make_external_call( 'mv toggle.js '+ opts.physdir, opts.debug, opts.debug,  True)

#let us close the log file
logfile.write("</log>")
logfile.close()
logfile = __name__+".xml"
output, status = make_external_call( 'grep WARNING '+ logfile +'| wc - | awk \'{print $1}\' - ', opts.debug, opts.debug, True)

if status==0:
  if int(output)==0:
    print 'No warnings'
  else:
    print '\n\n\nThere are warnings : '+str(int(output))+' . Check the log file '+logfile
  
  output, status = make_external_call('mv '+logfile + " "+opts.physdir, True,True,True) 
else:
  print 'Could not find the log file ' +logfile
  
 
#Finally create the xsl for the log xml file
logfile = open(__name__+".xsl", "w")
logfile.write("""<?xml version="1.0" encoding="ISO-8859-1"?>
<xsl:stylesheet version="1.0"
xmlns:xsl="http://www.w3.org/1999/XSL/Transform">

<xsl:template match="/">
  <html>
  <body>
  <center><h1>Log file summary </h1></center>
  <xsl:apply-templates/>
  </body>
  </html>
</xsl:template>

<xsl:template match="section">
<h2 color="blue">Section: <xsl:value-of select="."/></h2>
<br />
</xsl:template>

<xsl:template match="done">
<center>
  <div>
    <table bgcolor="green" border="2" width="80%">
      <tr>
        <td width="80%"><xsl:value-of select="."/></td>
        <td bgcolor="white"> passed</td>
      </tr>
    </table>
  </div>
</center>
</xsl:template>
<xsl:template match="warning">
<center>
  <div>
    <table bgcolor="orange" border="2" width="80%">
      <tr>
        <td width="80%"><xsl:value-of select="."/></td>
        <td bgcolor="white"> fix me</td>
      </tr>
    </table>
  </div>
</center>
</xsl:template>
<xsl:template match="error">
<center>
  <div>
    <table bgcolor="red" border="2" width="80%">
      <tr>
        <td width="80%"><xsl:value-of select="."/></td>
        <td bgcolor="white"> skipped</td>
      </tr>
    </table>
  </div>
</center>
</xsl:template>



</xsl:stylesheet>
""")

logfile.close()
output, status = make_external_call('mv '+__name__+".xsl" + " "+opts.physdir, True,True,True) 


