import ast
import base64
import cgi
import codecs
import collections
import copy
import datetime
from distutils.version import LooseVersion
import filecmp
from functools import update_wrapper
import hashlib
import importlib
import inspect
import io
from io import BytesIO, TextIOWrapper
import json
import logging
import math
import mimetypes
import operator
import os
import pickle
import re
import shutil
import socket
import stat
import subprocess
from subprocess import Popen, PIPE
import sys
import tarfile
import tempfile
import threading
import time
import traceback
import types
import unicodedata
import urllib
from urllib.parse import quote as urllibquote, quote_plus as urllibquoteplus
from urllib.parse import unquote as urllibunquote
from urllib.parse import urlparse, urlunparse, urlencode, urlsplit, parse_qsl
from urllib.request import urlretrieve
import uuid
import weakref
import xml.etree.ElementTree as ET
import zipfile

import docassemble.base.config
if not docassemble.base.config.loaded:
    docassemble.base.config.load()
from docassemble.base.config import daconfig, hostname, in_celery

import docassemble.webapp.setup
from docassemble.webapp.setup import da_version
import docassemble.base.astparser
from docassemble.base.error import DAError, DAErrorNoEndpoint, DAErrorMissingVariable, DAErrorCompileError, DAValidationError
import docassemble.base.functions
from docassemble.base.functions import pickleable_objects, word, comma_and_list, get_default_timezone, ReturnValue
from docassemble.base.generate_key import random_string, random_lower_string, random_alphanumeric, random_digits
import docassemble.base.interview_cache
from docassemble.base.logger import logmessage
from docassemble.base.pandoc import word_to_markdown, convertible_mimetypes, convertible_extensions
import docassemble.base.parse
import docassemble.base.pdftk
from docassemble.base.standardformatter import as_html, as_sms, get_choices_with_abb
import docassemble.base.util
from docassemble.base.util import DAEmail, DAEmailRecipientList, DAEmailRecipient, DAFileList, DAFile, DAObject, DAFileCollection, DAStaticFile, DADict, DAList
import docassemble.base.core # for backward-compatibility with data pickled in earlier versions

from docassemble.webapp.app_object import app, csrf, flaskbabel
import docassemble.webapp.backend
from docassemble.webapp.backend import cloud, initial_dict, can_access_file_number, get_info_from_file_number, get_info_from_file_number_with_uids, da_send_mail, get_new_file_number, pad, unpad, encrypt_phrase, pack_phrase, decrypt_phrase, unpack_phrase, encrypt_dictionary, pack_dictionary, decrypt_dictionary, unpack_dictionary, nice_date_from_utc, fetch_user_dict, fetch_previous_user_dict, advance_progress, reset_user_dict, get_chat_log, save_numbered_file, generate_csrf, get_info_from_file_reference, reference_exists, write_ml_source, fix_ml_files, is_package_ml, user_dict_exists, file_set_attributes, file_user_access, file_privilege_access, url_if_exists, get_person, Message, url_for, encrypt_object, decrypt_object, delete_user_data, delete_temp_user_data, clear_session, clear_specific_session, guess_yaml_filename, get_session, get_uid_for_filename, update_session, get_session_uids, project_name, directory_for, add_project
import docassemble.webapp.clicksend
from docassemble.webapp.core.models import Uploads, UploadsUserAuth, SpeakList, Supervisors, Shortener, Email, EmailAttachment, MachineLearning, GlobalObjectStorage
from docassemble.webapp.daredis import r, r_user, r_store
from docassemble.webapp.db_object import db
from docassemble.webapp.develop import CreatePackageForm, CreatePlaygroundPackageForm, UpdatePackageForm, ConfigForm, PlaygroundForm, PlaygroundUploadForm, LogForm, Utilities, PlaygroundFilesForm, PlaygroundFilesEditForm, PlaygroundPackagesForm, GoogleDriveForm, OneDriveForm, GitHubForm, PullPlaygroundPackage, TrainingForm, TrainingUploadForm, APIKey, AddinUploadForm, FunctionFileForm, RenameProject, DeleteProject, NewProject
from docassemble.webapp.files import SavedFile, get_ext_and_mimetype, make_package_zip
from docassemble.webapp.fixpickle import fix_pickle_obj
from docassemble.webapp.jsonstore import read_answer_json, write_answer_json, delete_answer_json, variables_snapshot_connection
import docassemble.webapp.machinelearning
from docassemble.webapp.packages.models import Package, PackageAuth, Install
from docassemble.webapp.playground import PlaygroundSection
from docassemble.webapp.screenreader import to_text
from docassemble.webapp.translations import setup_translation
from docassemble.webapp.users.forms import MyRegisterForm, MyInviteForm, MySignInForm, PhoneLoginForm, PhoneLoginVerifyForm, MFASetupForm, MFAReconfigureForm, MFALoginForm, MFAChooseForm, MFASMSSetupForm, MFAVerifySMSSetupForm, MyResendConfirmEmailForm, ManageAccountForm, RequestDeveloperForm
from docassemble.webapp.users.models import UserAuthModel, UserModel, UserDict, UserDictKeys, TempUser, ChatLog, MyUserInvitation, Role, UserRoles
from docassemble.webapp.users.views import user_profile_page
if not in_celery:
    import docassemble.webapp.worker

import apiclient
import babel.dates
from bs4 import BeautifulSoup
from Crypto.Hash import MD5
from Crypto.PublicKey import RSA
import dateutil
import dateutil.parser
from dateutil import tz
import docassemble_flask_user.emails
import docassemble_flask_user.forms
import docassemble_flask_user.signals
import docassemble_flask_user.views
from docassemble_flask_user import UserManager, SQLAlchemyAdapter
from docassemble_flask_user import login_required, roles_required, user_logged_in, user_changed_password, user_registered, user_reset_password
from docassemblekvsession import KVSessionExtension
from docassemble_textstat.textstat import textstat
from flask import make_response, abort, render_template, render_template_string, request, session, send_file, redirect, current_app, get_flashed_messages, flash, Markup, jsonify, Response, g
from flask_cors import cross_origin
from flask_login import LoginManager
from flask_login import login_user, logout_user, current_user
from flask_wtf.csrf import CSRFError
import googleapiclient.discovery
import httplib2
import humanize
from jinja2.exceptions import TemplateError
import links_from_header
import oauth2client.client
import pandas
from PIL import Image
import pkg_resources
import psutil
import pyotp
from pygments import highlight
from pygments.formatters import HtmlFormatter
from pygments.lexers import YamlLexer, PythonLexer
import pytz
import qrcode
import qrcode.image.svg
from rauth import OAuth1Service, OAuth2Service
import requests
import ruamel.yaml
from simplekv.memory.redisstore import RedisStore
from sqlalchemy import or_, and_, select, delete, update
import tailer
import twilio.twiml
import twilio.twiml.messaging_response
import twilio.twiml.voice_response
from twilio.rest import Client as TwilioRestClient
import werkzeug.exceptions
import werkzeug.utils
import wtforms
import xlsxwriter
from user_agents import parse as ua_parse
import yaml

docassemble.base.util.set_knn_machine_learner(docassemble.webapp.machinelearning.SimpleTextMachineLearner)
docassemble.base.util.set_machine_learning_entry(docassemble.webapp.machinelearning.MachineLearningEntry)
docassemble.base.util.set_random_forest_machine_learner(docassemble.webapp.machinelearning.RandomForestMachineLearner)
docassemble.base.util.set_svm_machine_learner(docassemble.webapp.machinelearning.SVMMachineLearner)

START_TIME = time.time()

min_system_version = '1.2.0'
re._MAXCACHE = 10000

the_method_type = types.FunctionType
equals_byte = bytes('=', 'utf-8')

TypeType = type(type(None))
NoneType = type(None)

STATS = daconfig.get('collect statistics', False)
DEBUG = daconfig.get('debug', False)
ERROR_TYPES_NO_EMAIL = daconfig.get('suppress error notificiations', [])
COOKIELESS_SESSIONS = daconfig.get('cookieless sessions', False)

if DEBUG:
    PREVENT_DEMO = False
elif daconfig.get('allow demo', False):
    PREVENT_DEMO = False
else:
    PREVENT_DEMO = True

REQUIRE_IDEMPOTENT = not daconfig.get('allow non-idempotent questions', True)
STRICT_MODE = daconfig.get('restrict input variables', False)
PACKAGE_PROTECTION = daconfig.get('package protection', True)

HTTP_TO_HTTPS = daconfig.get('behind https load balancer', False)
GITHUB_BRANCH = daconfig.get('github default branch name', 'main')
request_active = True

global_css = ''

global_js = ''

default_playground_yaml = """metadata:
  title: Default playground interview
  short title: Test
  comment: This is a learning tool.  Feel free to write over it.
---
objects:
  - client: Individual
---
question: |
  What is your name?
fields:
  - First Name: client.name.first
  - Middle Name: client.name.middle
    required: False
  - Last Name: client.name.last
  - Suffix: client.name.suffix
    required: False
    code: name_suffix()
---
question: |
  What is your date of birth?
fields:
  - Date of Birth: client.birthdate
    datatype: date
---
mandatory: True
question: |
  Here is your document, ${ client }.
subquestion: |
  In order ${ quest }, you will need this.
attachments:
  - name: Information Sheet
    filename: info_sheet
    content: |
      Your name is ${ client }.

      % if client.age_in_years() > 60:
      You are a senior.
      % endif
      Your quest is ${ quest }.  You
      are eligible for ${ benefits }.
---
question: |
  What is your quest?
fields:
  - Your quest: quest
    hint: to find the Loch Ness Monster
---
code: |
  if client.age_in_years() < 18:
    benefits = "CHIP"
  else:
    benefits = "Medicaid"
"""

ok_mimetypes = {
    "application/javascript": "javascript",
    "application/json": "javascript",
    "text/css": "css",
    "text/html": "htmlmixed",
    "text/x-python": "python"
}
ok_extensions = {
    "4th": "forth",
    "apl": "apl",
    "asc": "asciiarmor",
    "asn": "asn.1",
    "asn1": "asn.1",
    "aspx": "htmlembedded",
    "b": "brainfuck",
    "bash": "shell",
    "bf": "brainfuck",
    "c": "clike",
    "c++": "clike",
    "cc": "clike",
    "cl": "commonlisp",
    "clj": "clojure",
    "cljc": "clojure",
    "cljs": "clojure",
    "cljx": "clojure",
    "cob": "cobol",
    "coffee": "coffeescript",
    "cpp": "clike",
    "cpy": "cobol",
    "cql": "sql",
    "cr": "crystal",
    "cs": "clike",
    "csharp": "clike",
    "css": "css",
    "cxx": "clike",
    "cyp": "cypher",
    "cypher": "cypher",
    "d": "d",
    "dart": "dart",
    "diff": "diff",
    "dtd": "dtd",
    "dyalog": "apl",
    "dyl": "dylan",
    "dylan": "dylan",
    "e": "eiffel",
    "ecl": "ecl",
    "ecmascript": "javascript",
    "edn": "clojure",
    "ejs": "htmlembedded",
    "el": "commonlisp",
    "elm": "elm",
    "erb": "htmlembedded",
    "erl": "erlang",
    "f": "fortran",
    "f77": "fortran",
    "f90": "fortran",
    "f95": "fortran",
    "factor": "factor",
    "feature": "gherkin",
    "for": "fortran",
    "forth": "forth",
    "fs": "mllike",
    "fth": "forth",
    "fun": "mllike",
    "go": "go",
    "gradle": "groovy",
    "groovy": "groovy",
    "gss": "css",
    "h": "clike",
    "h++": "clike",
    "haml": "haml",
    "handlebars": "htmlmixed",
    "hbs": "htmlmixed",
    "hh": "clike",
    "hpp": "clike",
    "hs": "haskell",
    "html": "htmlmixed",
    "hx": "haxe",
    "hxml": "haxe",
    "hxx": "clike",
    "in": "properties",
    "ini": "properties",
    "ino": "clike",
    "intr": "dylan",
    "j2": "jinja2",
    "jade": "pug",
    "java": "clike",
    "jinja": "jinja2",
    "jinja2": "jinja2",
    "jl": "julia",
    "json": "json",
    "jsonld": "javascript",
    "jsp": "htmlembedded",
    "jsx": "jsx",
    "ksh": "shell",
    "kt": "clike",
    "less": "css",
    "lhs": "haskell-literate",
    "lisp": "commonlisp",
    "ls": "livescript",
    "ltx": "stex",
    "lua": "lua",
    "m": "octave",
    "markdown": "markdown",
    "mbox": "mbox",
    "md": "markdown",
    "mkd": "markdown",
    "mo": "modelica",
    "mps": "mumps",
    "msc": "mscgen",
    "mscgen": "mscgen",
    "mscin": "mscgen",
    "msgenny": "mscgen",
    "node": "javascript",
    "nq": "ntriples",
    "nsh": "nsis",
    "nsi": "nsis",
    "nt": "ntriples",
    "nut": "clike",
    "oz": "oz",
    "p": "pascal",
    "pas": "pascal",
    "patch": "diff",
    "pgp": "asciiarmor",
    "php": "php",
    "php3": "php",
    "php4": "php",
    "php5": "php",
    "php7": "php",
    "phtml": "php",
    "pig": "pig",
    "pl": "perl",
    "pls": "sql",
    "pm": "perl",
    "pp": "puppet",
    "pro": "idl",
    "properties": "properties",
    "proto": "protobuf",
    "ps1": "powershell",
    "psd1": "powershell",
    "psm1": "powershell",
    "pug": "pug",
    "pxd": "python",
    "pxi": "python",
    "py": "python",
    "pyx": "python",
    "q": "q",
    "r": "r",
    "rb": "ruby",
    "rq": "sparql",
    "rs": "rust",
    "rst": "rst",
    "s": "gas",
    "sas": "sas",
    "sass": "sass",
    "scala": "clike",
    "scm": "scheme",
    "scss": "css",
    "sh": "shell",
    "sieve": "sieve",
    "sig": "asciiarmor",
    "siv": "sieve",
    "slim": "slim",
    "smackspec": "mllike",
    "sml": "mllike",
    "soy": "soy",
    "sparql": "sparql",
    "sql": "sql",
    "ss": "scheme",
    "st": "smalltalk",
    "styl": "stylus",
    "swift": "swift",
    "tcl": "tcl",
    "tex": "stex",
    "textile": "textile",
    "toml": "toml",
    "tpl": "smarty",
    "ts": "javascript",
    "tsx": "javascript",
    "ttcn": "ttcn",
    "ttcn3": "ttcn",
    "ttcnpp": "ttcn",
    "ttl": "turtle",
    "vb": "vb",
    "vbs": "vbscript",
    "vhd": "vhdl",
    "vhdl": "vhdl",
    "vtl": "velocity",
    "vue": "vue",
    "wast": "wast",
    "wat": "wast",
    "webidl": "webidl",
    "xml": "xml",
    "xquery": "xquery",
    "xsd": "xml",
    "xsl": "xml",
    "xu": "mscgen",
    "xy": "xquery",
    "yaml": "yaml",
    "yml": "yaml",
    "ys": "yacas",
    "z80": "z80"
}

def update_editable():
    try:
        if 'editable mimetypes' in daconfig and isinstance(daconfig['editable mimetypes'], list):
            for item in daconfig['editable mimetypes']:
                if isinstance(item, str):
                    ok_mimetypes[item] = 'null'
    except:
        pass

    try:
        if 'editable extensions' in daconfig and isinstance(daconfig['editable extensions'], list):
            for item in daconfig['editable extensions']:
                if isinstance(item, str):
                    ok_extensions[item] = 'null'
    except:
        pass

update_editable()

default_yaml_filename = daconfig.get('default interview', None)
final_default_yaml_filename = daconfig.get('default interview', 'docassemble.base:data/questions/default-interview.yml')
keymap = daconfig.get('keymap', None)
google_config = daconfig.get('google', {})

contains_volatile = re.compile(r'^(x\.|x\[|.*\[[ijklmn]\])')
is_integer = re.compile(r'[0-9]+')
detect_mobile = re.compile(r'Mobile|iP(hone|od|ad)|Android|BlackBerry|IEMobile|Kindle|NetFront|Silk-Accelerated|(hpw|web)OS|Fennec|Minimo|Opera M(obi|ini)|Blazer|Dolfin|Dolphin|Skyfire|Zune')
alphanumeric_only = re.compile(r'[\W_]+')
phone_pattern = re.compile(r"^[\d\+\-\(\) ]+$")
document_match = re.compile(r'^--- *$', flags=re.MULTILINE)
fix_tabs = re.compile(r'\t')
fix_initial = re.compile(r'^---\n')
noquote_match = re.compile(r'"')
lt_match = re.compile(r'<')
gt_match = re.compile(r'>')
amp_match = re.compile(r'&')
extraneous_var = re.compile(r'^x\.|^x\[')
key_requires_preassembly = re.compile(r'^(session_local\.|device_local\.|user_local\.|x\.|x\[|_multiple_choice|.*\[[ijklmn]\])')
#match_invalid = re.compile('[^A-Za-z0-9_\[\].\'\%\-=]')
#match_invalid_key = re.compile('[^A-Za-z0-9_\[\].\'\%\- =]')
match_brackets = re.compile(r'\[[BR]?\'[^\]]*\'\]$')
match_inside_and_outside_brackets = re.compile(r'(.*)(\[[BR]?\'[^\]]*\'\])$')
match_inside_brackets = re.compile(r'\[([BR]?)\'([^\]]*)\'\]')
valid_python_var = re.compile(r'[A-Za-z][A-Za-z0-9\_]*')
valid_python_exp = re.compile(r'[A-Za-z][A-Za-z0-9\_\.]*')

default_title = daconfig.get('default title', daconfig.get('brandname', 'docassemble'))
default_short_title = daconfig.get('default short title', default_title)
os.environ['PYTHON_EGG_CACHE'] = tempfile.gettempdir()
PNG_RESOLUTION = daconfig.get('png resolution', 300)
PNG_SCREEN_RESOLUTION = daconfig.get('png screen resolution', 72)
PDFTOPPM_COMMAND = daconfig.get('pdftoppm', 'pdftoppm')
DEFAULT_LANGUAGE = daconfig.get('language', 'en')
DEFAULT_LOCALE = daconfig.get('locale', 'en_US.utf8')
DEFAULT_DIALECT = daconfig.get('dialect', 'us')
LOGSERVER = daconfig.get('log server', None)
CHECKIN_INTERVAL = int(daconfig.get('checkin interval', 6000))
#message_sequence = dbtableprefix + 'message_id_seq'
NOTIFICATION_CONTAINER = '<div class="datopcenter col-sm-7 col-md-6 col-lg-5" id="daflash">%s</div>'
NOTIFICATION_MESSAGE = '<div class="da-alert alert alert-%s alert-dismissible fade show" role="alert">%s<button type="button" class="btn-close" data-bs-dismiss="alert" aria-label="Close"></button></div>'

USING_SUPERVISOR = bool(os.environ.get('SUPERVISOR_SERVER_URL', None))

audio_mimetype_table = {'mp3': 'audio/mpeg', 'ogg': 'audio/ogg'}

valid_voicerss_dialects = {
    'ca': ['es'],
    'zh': ['cn', 'hk', 'tw'],
    'da': ['dk'],
    'nl': ['nl'],
    'en': ['au', 'ca', 'gb', 'in', 'us'],
    'fi': ['fi'],
    'fr': ['ca, fr'],
    'de': ['de'],
    'it': ['it'],
    'ja': ['jp'],
    'ko': ['kr'],
    'nb': ['no'],
    'pl': ['pl'],
    'pt': ['br', 'pt'],
    'ru': ['ru'],
    'es': ['mx', 'es'],
    'sv': ['se']
    }

voicerss_config = daconfig.get('voicerss', None)
VOICERSS_ENABLED = not bool(not voicerss_config or ('enable' in voicerss_config and not voicerss_config['enable']) or not ('key' in voicerss_config and voicerss_config['key']))
ROOT = daconfig.get('root', '/')
#app.logger.warning("default sender is " + current_app.config['MAIL_DEFAULT_SENDER'] + "\n")
exit_page = daconfig.get('exitpage', 'https://docassemble.org')

SUPERVISORCTL = daconfig.get('supervisorctl', 'supervisorctl')
#PACKAGE_CACHE = daconfig.get('packagecache', '/var/www/.cache')
WEBAPP_PATH = daconfig.get('webapp', '/usr/share/docassemble/webapp/docassemble.wsgi')
UPLOAD_DIRECTORY = daconfig.get('uploads', '/usr/share/docassemble/files')
PACKAGE_DIRECTORY = daconfig.get('packages', '/usr/share/docassemble/local' + str(sys.version_info.major) + '.' + str(sys.version_info.minor))
FULL_PACKAGE_DIRECTORY = os.path.join(PACKAGE_DIRECTORY, 'lib', 'python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor), 'site-packages')
LOG_DIRECTORY = daconfig.get('log', '/usr/share/docassemble/log')

PAGINATION_LIMIT = daconfig.get('pagination limit', 100)
PAGINATION_LIMIT_PLUS_ONE = PAGINATION_LIMIT + 1

#PLAYGROUND_MODULES_DIRECTORY = daconfig.get('playground_modules', )

init_py_file = """try:
    __import__('pkg_resources').declare_namespace(__name__)
except ImportError:
    __path__ = __import__('pkgutil').extend_path(__path__, __name__)
"""

#if not os.path.isfile(os.path.join(PLAYGROUND_MODULES_DIRECTORY, 'docassemble', '__init__.py')):
#    with open(os.path.join(PLAYGROUND_MODULES_DIRECTORY, 'docassemble', '__init__.py'), 'a') as the_file:
#        the_file.write(init_py_file)

#USE_PROGRESS_BAR = daconfig.get('use_progress_bar', True)
SHOW_LOGIN = daconfig.get('show login', True)
ALLOW_REGISTRATION = daconfig.get('allow registration', True)
#USER_PACKAGES = daconfig.get('user_packages', '/var/lib/docassemble/dist-packages')
#sys.path.append(USER_PACKAGES)
#if USE_PROGRESS_BAR:

if in_celery:
    LOGFILE = daconfig.get('celery flask log', '/tmp/celery-flask.log')
else:
    LOGFILE = daconfig.get('flask log', '/tmp/flask.log')
#APACHE_LOGFILE = daconfig.get('apache_log', '/var/log/apache2/error.log')

#connect_string = docassemble.webapp.database.connection_string()
#alchemy_connect_string = docassemble.webapp.database.alchemy_connection_string()

mimetypes.add_type('application/x-yaml', '.yml')
mimetypes.add_type('application/x-yaml', '.yaml')

store = RedisStore(r_store)

kv_session = KVSessionExtension(store, app)

def _call_or_get(function_or_property):
    return function_or_property() if callable(function_or_property) else function_or_property

def _get_safe_next_param(param_name, default_endpoint):
    if param_name in request.args:
        safe_next = current_app.user_manager.make_safe_url_function(urllibunquote(request.args[param_name]))
        #safe_next = request.args[param_name]
    else:
        safe_next = _endpoint_url(default_endpoint)
    return safe_next

# def _do_login_user(user, safe_next, remember_me=False):
#     if not user: return unauthenticated()

#     if not _call_or_get(user.is_active):
#         flash(word('Your account has not been enabled.'), 'error')
#         return redirect(url_for('user.login'))

#     user_manager = current_app.user_manager
#     if user_manager.enable_email and user_manager.enable_confirm_email \
#             and not current_app.user_manager.enable_login_without_confirm_email \
#             and not user.has_confirmed_email():
#         url = url_for('user.resend_confirm_email')
#         flash(docassemble_flask_user.translations.gettext('Your email address has not yet been confirmed. Check your email Inbox and Spam folders for the confirmation email or <a href="%(url)s">Re-send confirmation email</a>.', url=url), 'error')
#         return redirect(url_for('user.login'))

#     login_user(user, remember=remember_me)

#     signals.user_logged_in.send(current_app._get_current_object(), user=user)

#     flash(word('You have signed in successfully.'), 'success')

#     return redirect(safe_next)

def custom_resend_confirm_email():
    user_manager = current_app.user_manager
    form = user_manager.resend_confirm_email_form(request.form)
    if request.method=='GET' and 'email' in request.args:
        form.email.data = request.args['email']
    if request.method=='POST' and form.validate():
        email = form.email.data
        user, user_email = user_manager.find_user_by_email(email)
        if user:
            docassemble_flask_user.views._send_confirm_email(user, user_email)
        return redirect(docassemble_flask_user.views._endpoint_url(user_manager.after_resend_confirm_email_endpoint))
    response = make_response(user_manager.render_function(user_manager.resend_confirm_email_template, form=form), 200)
    response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
    return response

def as_int(val):
    try:
        return int(val)
    except:
        return 0

def custom_register():
    """Display registration form and create new User."""
    is_json = bool(('json' in request.form and as_int(request.form['json'])) or ('json' in request.args and as_int(request.args['json'])))

    user_manager =  current_app.user_manager
    db_adapter = user_manager.db_adapter

    safe_next = _get_safe_next_param('next', user_manager.after_login_endpoint)
    safe_reg_next = _get_safe_next_param('reg_next', user_manager.after_register_endpoint)
    if _call_or_get(current_user.is_authenticated) and user_manager.auto_login_at_login:
        if safe_next == url_for(user_manager.after_login_endpoint):
            url_parts = list(urlparse(safe_next))
            query = dict(parse_qsl(url_parts[4]))
            query.update(dict(from_login=1))
            url_parts[4] = urlencode(query)
            safe_next = urlunparse(url_parts)
        return add_secret_to(redirect(safe_next))

    setup_translation()

    # Initialize form
    login_form = user_manager.login_form()                      # for login_or_register.html
    register_form = user_manager.register_form(request.form)    # for register.html

    # invite token used to determine validity of registeree
    invite_token = request.values.get("token")

    # require invite without a token should disallow the user from registering
    if user_manager.require_invitation and not invite_token:
        flash(word("Registration is invite only"), "error")
        return redirect(url_for('user.login'))

    user_invite = None
    if invite_token and db_adapter.UserInvitationClass:
        user_invite = db_adapter.find_first_object(db_adapter.UserInvitationClass, token=invite_token)
        if user_invite:
            register_form.invite_token.data = invite_token
        else:
            flash(word("Invalid invitation token"), "error")
            return redirect(url_for('user.login'))

    if request.method != 'POST':
        login_form.next.data     = register_form.next.data     = safe_next
        login_form.reg_next.data = register_form.reg_next.data = safe_reg_next
        if user_invite:
            register_form.email.data = user_invite.email

    # Process valid POST
    if request.method == 'POST' and register_form.validate():
        email_taken = False
        if daconfig.get('confirm registration', False):
            try:
                docassemble_flask_user.forms.unique_email_validator(register_form, register_form.email)
            except wtforms.ValidationError:
                email_taken = True
        if email_taken:
            flash(word('A confirmation email has been sent to %(email)s with instructions to complete your registration.' % {'email': register_form.email.data}), 'success')
            subject, html_message, text_message = docassemble_flask_user.emails._render_email(
                'flask_user/emails/reregistered',
                app_name=app.config['APP_NAME'],
                sign_in_link=url_for('user.login', _external=True))

            # Send email message using Flask-Mail
            user_manager.send_email_function(register_form.email.data, subject, html_message, text_message)
            return redirect(url_for('user.login'))

        # Create a User object using Form fields that have a corresponding User field
        User = db_adapter.UserClass
        user_class_fields = User.__dict__
        user_fields = {}

        # Create a UserEmail object using Form fields that have a corresponding UserEmail field
        if db_adapter.UserEmailClass:
            UserEmail = db_adapter.UserEmailClass
            user_email_class_fields = UserEmail.__dict__
            user_email_fields = {}

        # Create a UserAuth object using Form fields that have a corresponding UserAuth field
        if db_adapter.UserAuthClass:
            UserAuth = db_adapter.UserAuthClass
            user_auth_class_fields = UserAuth.__dict__
            user_auth_fields = {}

        # Enable user account
        if db_adapter.UserProfileClass:
            if hasattr(db_adapter.UserProfileClass, 'active'):
                user_auth_fields['active'] = True
            elif hasattr(db_adapter.UserProfileClass, 'is_enabled'):
                user_auth_fields['is_enabled'] = True
            else:
                user_auth_fields['is_active'] = True
        else:
            if hasattr(db_adapter.UserClass, 'active'):
                user_fields['active'] = True
            elif hasattr(db_adapter.UserClass, 'is_enabled'):
                user_fields['is_enabled'] = True
            else:
                user_fields['is_active'] = True

        # For all form fields
        for field_name, field_value in register_form.data.items():
            # Hash password field
            if field_name == 'password':
                hashed_password = user_manager.hash_password(field_value)
                if db_adapter.UserAuthClass:
                    user_auth_fields['password'] = hashed_password
                else:
                    user_fields['password'] = hashed_password
            # Store corresponding Form fields into the User object and/or UserProfile object
            else:
                if field_name in user_class_fields:
                    user_fields[field_name] = field_value
                if db_adapter.UserEmailClass:
                    if field_name in user_email_class_fields:
                        user_email_fields[field_name] = field_value
                if db_adapter.UserAuthClass:
                    if field_name in user_auth_class_fields:
                        user_auth_fields[field_name] = field_value

        # Add User record using named arguments 'user_fields'
        user = db_adapter.add_object(User, **user_fields)

        # Add UserEmail record using named arguments 'user_email_fields'
        if db_adapter.UserEmailClass:
            user_email = db_adapter.add_object(UserEmail,
                    user=user,
                    is_primary=True,
                    **user_email_fields)
        else:
            user_email = None

        # Add UserAuth record using named arguments 'user_auth_fields'
        if db_adapter.UserAuthClass:
            user_auth = db_adapter.add_object(UserAuth, **user_auth_fields)
            if db_adapter.UserProfileClass:
                user = user_auth
            else:
                user.user_auth = user_auth

        require_email_confirmation = True
        if user_invite:
            if user_invite.email == register_form.email.data:
                require_email_confirmation = False
                db_adapter.update_object(user, confirmed_at=datetime.datetime.utcnow())

        db_adapter.commit()

        # Send 'registered' email and delete new User object if send fails
        if user_manager.send_registered_email:
            try:
                # Send 'registered' email
                docassemble_flask_user.views._send_registered_email(user, user_email, require_email_confirmation)
            except Exception:
                # delete new User object if send fails
                db_adapter.delete_object(user)
                db_adapter.commit()
                raise

        # Send user_registered signal
        docassemble_flask_user.signals.user_registered.send(current_app._get_current_object(),
                                                user=user,
                                                user_invite=user_invite)

        # Redirect if USER_ENABLE_CONFIRM_EMAIL is set
        if user_manager.enable_confirm_email and require_email_confirmation:
            safe_reg_next = user_manager.make_safe_url_function(register_form.reg_next.data)
            return redirect(safe_reg_next)

        # Auto-login after register or redirect to login page
        if 'reg_next' in request.args:
            safe_reg_next = user_manager.make_safe_url_function(register_form.reg_next.data)
        else:
            safe_reg_next = _endpoint_url(user_manager.after_confirm_endpoint)

        if user_manager.auto_login_after_register:
            if app.config['USE_MFA']:
                if user.otp_secret is None and len(app.config['MFA_REQUIRED_FOR_ROLE']) and user.has_role(*app.config['MFA_REQUIRED_FOR_ROLE']):
                    session['validated_user'] = user.id
                    session['next'] = safe_reg_next
                    if app.config['MFA_ALLOW_APP'] and (twilio_config is None or not app.config['MFA_ALLOW_SMS']):
                        return redirect(url_for('mfa_setup'))
                    if not app.config['MFA_ALLOW_APP']:
                        return redirect(url_for('mfa_sms_setup'))
                    return redirect(url_for('mfa_choose'))
            return docassemble_flask_user.views._do_login_user(user, safe_reg_next)
        return redirect(url_for('user.login') + '?next=' + urllibquote(safe_reg_next))

    # Process GET or invalid POST
    if is_json:
        return jsonify(action='register', csrf_token=generate_csrf())
    response = make_response(user_manager.render_function(user_manager.register_template,
                                                              form=register_form,
                                                              login_form=login_form,
                                                              register_form=register_form), 200)
    response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
    return response

def custom_login():
    """ Prompt for username/email and password and sign the user in."""
    #sys.stderr.write("In custom_login\n")
    #logmessage("Doing custom_login")

    is_json = bool(('json' in request.form and as_int(request.form['json'])) or ('json' in request.args and as_int(request.args['json'])))
    user_manager = current_app.user_manager
    db_adapter = user_manager.db_adapter

    safe_next = _get_safe_next_param('next', user_manager.after_login_endpoint)
    safe_reg_next = _get_safe_next_param('reg_next', user_manager.after_register_endpoint)
    if safe_next and '/officeaddin' in safe_next:
        g.embed = True

    if _call_or_get(current_user.is_authenticated) and user_manager.auto_login_at_login:
        if safe_next == url_for(user_manager.after_login_endpoint):
            url_parts = list(urlparse(safe_next))
            query = dict(parse_qsl(url_parts[4]))
            query.update(dict(from_login=1))
            url_parts[4] = urlencode(query)
            safe_next = urlunparse(url_parts)
        return add_secret_to(redirect(safe_next))

    setup_translation()

    login_form = user_manager.login_form(request.form)
    register_form = user_manager.register_form()
    if request.method != 'POST':
        login_form.next.data     = register_form.next.data     = safe_next
        login_form.reg_next.data = register_form.reg_next.data = safe_reg_next
    if request.method == 'GET' and 'validated_user' in session:
        del session['validated_user']
    if request.method=='POST' and login_form.validate():
        user = None
        user_email = None
        if user_manager.enable_username:
            user = user_manager.find_user_by_username(login_form.username.data)
            user_email = None
            if user and db_adapter.UserEmailClass:
                user_email = db_adapter.find_first_object(db_adapter.UserEmailClass,
                        user_id=int(user.get_id()),
                        is_primary=True,
                        )
            if not user and user_manager.enable_email:
                user, user_email = user_manager.find_user_by_email(login_form.username.data)
        else:
            user, user_email = user_manager.find_user_by_email(login_form.email.data)
        #if not user and daconfig['ldap login'].get('enabled', False):
        if user:
            safe_next = user_manager.make_safe_url_function(login_form.next.data)
            #safe_next = login_form.next.data
            #safe_next = url_for('post_login', next=login_form.next.data)
            if app.config['USE_MFA']:
                if user.otp_secret is None and len(app.config['MFA_REQUIRED_FOR_ROLE']) and user.has_role(*app.config['MFA_REQUIRED_FOR_ROLE']):
                    session['validated_user'] = user.id
                    session['next'] = safe_next
                    if app.config['MFA_ALLOW_APP'] and (twilio_config is None or not app.config['MFA_ALLOW_SMS']):
                        return redirect(url_for('mfa_setup'))
                    if not app.config['MFA_ALLOW_APP']:
                        return redirect(url_for('mfa_sms_setup'))
                    return redirect(url_for('mfa_choose'))
                if user.otp_secret is not None:
                    session['validated_user'] = user.id
                    session['next'] = safe_next
                    if user.otp_secret.startswith(':phone:'):
                        phone_number = re.sub(r'^:phone:', '', user.otp_secret)
                        verification_code = random_digits(daconfig['verification code digits'])
                        message = word("Your verification code is") + " " + str(verification_code) + "."
                        key = 'da:mfa:phone:' + str(phone_number) + ':code'
                        pipe = r.pipeline()
                        pipe.set(key, verification_code)
                        pipe.expire(key, daconfig['verification code timeout'])
                        pipe.execute()
                        success = docassemble.base.util.send_sms(to=phone_number, body=message)
                        if not success:
                            flash(word("Unable to send verification code."), 'error')
                            return redirect(url_for('user.login'))
                    return add_secret_to(redirect(url_for('mfa_login')))
            if user_manager.enable_email and user_manager.enable_confirm_email \
               and len(daconfig['email confirmation privileges']) \
               and user.has_role(*daconfig['email confirmation privileges']) \
               and not user.has_confirmed_email():
                url = url_for('user.resend_confirm_email', email=user.email)
                flash(word('You cannot log in until your e-mail address has been confirmed.') + '<br><a href="' + url + '">' + word('Click here to confirm your e-mail') + '</a>.', 'error')
                return redirect(url_for('user.login'))
            return add_secret_to(docassemble_flask_user.views._do_login_user(user, safe_next, login_form.remember_me.data))
    if is_json:
        return jsonify(action='login', csrf_token=generate_csrf())
    # if 'officeaddin' in safe_next:
    #     extra_css = """
    # <script type="text/javascript" src="https://appsforoffice.microsoft.com/lib/1.1/hosted/office.debug.js"></script>"""
    #     extra_js = """
    # <script type="text/javascript" src=""" + '"' + url_for('static', filename='office/fabric.js') + '"' + """></script>
    # <script type="text/javascript" src=""" + '"' + url_for('static', filename='office/polyfill.js') + '"' + """></script>
    # <script type="text/javascript" src=""" + '"' + url_for('static', filename='office/app.js') + '"' + """></script>"""
    #     return render_template(user_manager.login_template,
    #                            form=login_form,
    #                            login_form=login_form,
    #                            register_form=register_form,
    #                            extra_css=Markup(extra_css),
    #                            extra_js=Markup(extra_js))
    # else:
    response = make_response(user_manager.render_function(user_manager.login_template,
                                                          form=login_form,
                                                          login_form=login_form,
                                                          register_form=register_form), 200)
    response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
    return response

def add_secret_to(response):
    if 'newsecret' in session:
        if 'embed' in g:
            response.set_cookie('secret', session['newsecret'], httponly=True, secure=app.config['SESSION_COOKIE_SECURE'], samesite='None')
        else:
            response.set_cookie('secret', session['newsecret'], httponly=True, secure=app.config['SESSION_COOKIE_SECURE'], samesite=app.config['SESSION_COOKIE_SAMESITE'])
        del session['newsecret']
    return response

def logout():
    # secret = request.cookies.get('secret', None)
    # if secret is None:
    #     secret = random_string(16)
    #     set_cookie = True
    # else:
    #     secret = str(secret)
    #     set_cookie = False
    user_manager = current_app.user_manager
    if 'next' in request.args:
        next_url = request.args['next']
    elif session.get('language', None) and session['language'] != DEFAULT_LANGUAGE:
        next_url = _endpoint_url(user_manager.after_logout_endpoint, lang=session['language'])
    else:
        next_url = _endpoint_url(user_manager.after_logout_endpoint)
    if current_user.is_authenticated:
        if current_user.social_id.startswith('auth0$') and 'oauth' in daconfig and 'auth0' in daconfig['oauth'] and 'domain' in daconfig['oauth']['auth0']:
            if next_url.startswith('/'):
                next_url = get_base_url() + next_url
            next_url = 'https://' + daconfig['oauth']['auth0']['domain'] + '/v2/logout?' + urlencode(dict(returnTo=next_url, client_id=daconfig['oauth']['auth0']['id']))
        if current_user.social_id.startswith('keycloak$') and 'oauth' in daconfig and 'keycloak' in daconfig['oauth'] and 'domain' in daconfig['oauth']['keycloak']:
            if next_url.startswith('/'):
                next_url = get_base_url() + next_url
            next_url = ('https://' + daconfig['oauth']['keycloak']['domain'] + '/auth/realms/' + daconfig['oauth']['keycloak']['realm'] + '/protocol/openid-connect/logout?' + urlencode(dict(post_logout_redirect_uri=next_url))
            )
    else:
        if session.get('language', None) and session['language'] != DEFAULT_LANGUAGE:
            next_url = _endpoint_url(user_manager.after_logout_endpoint, lang=session['language'])
        else:
            next_url = _endpoint_url(user_manager.after_logout_endpoint)
    docassemble_flask_user.signals.user_logged_out.send(current_app._get_current_object(), user=current_user)
    logout_user()
    delete_session_info()
    session.clear()
    flash(word('You have signed out successfully.'), 'success')
    response = redirect(next_url)
    response.set_cookie('remember_token', '', expires=0)
    response.set_cookie('visitor_secret', '', expires=0)
    response.set_cookie('secret', '', expires=0)
    response.set_cookie('session', '', expires=0)
    return response

# def custom_login():
#     logmessage("custom_login")
#     user_manager = current_app.user_manager
#     db_adapter = user_manager.db_adapter
#     secret = request.cookies.get('secret', None)
#     if secret is not None:
#         secret = str(secret)
#     next_url = request.args.get('next', _endpoint_url(user_manager.after_login_endpoint))
#     reg_next = request.args.get('reg_next', _endpoint_url(user_manager.after_register_endpoint))

#     if _call_or_get(current_user.is_authenticated) and user_manager.auto_login_at_login:
#         return redirect(next_url)

#     login_form = user_manager.login_form(request.form)
#     register_form = user_manager.register_form()
#     if request.method != 'POST':
#         login_form.next.data     = register_form.next.data = next_url
#         login_form.reg_next.data = register_form.reg_next.data = reg_next

#     if request.method == 'POST':
#         try:
#             login_form.validate()
#         except:
#             logmessage("custom_login: got an error when validating login")
#             pass
#     if request.method == 'POST' and login_form.validate():
#         user = None
#         user_email = None
#         if user_manager.enable_username:
#             user = user_manager.find_user_by_username(login_form.username.data)
#             user_email = None
#             if user and db_adapter.UserEmailClass:
#                 user_email = db_adapter.find_first_object(db_adapter.UserEmailClass,
#                         user_id=int(user.get_id()),
#                         is_primary=True,
#                         )
#             if not user and user_manager.enable_email:
#                 user, user_email = user_manager.find_user_by_email(login_form.username.data)
#         else:
#             user, user_email = user_manager.find_user_by_email(login_form.email.data)

#         if user:
#             return _do_login_user(user, login_form.password.data, secret, login_form.next.data, login_form.remember_me.data)

#     return render_template(user_manager.login_template, page_title=word('Sign In'), tab_title=word('Sign In'), form=login_form, login_form=login_form, register_form=register_form)

def unauthenticated():
    if not request.args.get('nm', False):
        flash(word("You need to log in before you can access") + " " + word(request.path), 'error')
    the_url = url_for('user.login', next=fix_http(request.url))
    return redirect(the_url)

def unauthorized():
    flash(word("You are not authorized to access") + " " + word(request.path), 'error')
    return redirect(url_for('interview_list', next=fix_http(request.url)))

def my_default_url(error, endpoint, values):
    return url_for('index')

def make_safe_url(url):
    parts = urlsplit(url)
    safe_url = parts.path
    if parts.query != '':
        safe_url += '?' + parts.query
    if parts.fragment != '':
        safe_url += '#' + parts.fragment
    return safe_url

def password_validator(form, field):
    password = list(field.data)
    password_length = len(password)

    lowers = uppers = digits = punct = 0
    for ch in password:
        if ch.islower():
            lowers += 1
        if ch.isupper():
            uppers += 1
        if ch.isdigit():
            digits += 1
        if not (ch.islower() or ch.isupper() or ch.isdigit()):
            punct += 1

    rules = daconfig.get('password complexity', {})
    is_valid = password_length >= rules.get('length', 6) and lowers >= rules.get('lowercase', 1) and uppers >= rules.get('uppercase', 1) and digits >= rules.get('digits', 1) and punct >= rules.get('punctuation', 0)
    if not is_valid:
        if 'error message' in rules:
            error_message = str(rules['error message'])
        else:
            error_message = 'Password must be at least ' + docassemble.base.functions.quantity_noun(rules.get('length', 6), 'character', language='en') + ' long'
            standards = []
            if rules.get('lowercase', 1) > 0:
                standards.append('at least ' + docassemble.base.functions.quantity_noun(rules.get('lowercase', 1), 'lowercase letter', language='en'))
            if rules.get('uppercase', 1) > 0:
                standards.append('at least ' + docassemble.base.functions.quantity_noun(rules.get('uppercase', 1), 'uppercase letter', language='en'))
            if rules.get('digits', 1) > 0:
                standards.append('at least ' + docassemble.base.functions.quantity_noun(rules.get('digits', 1), 'number', language='en'))
            if rules.get('punctuation', 0) > 0:
                standards.append('at least ' + docassemble.base.functions.quantity_noun(rules.get('punctuation', 1), 'punctuation character', language='en'))
            if len(standards) > 0:
                error_message += ' with ' + docassemble.base.functions.comma_and_list_en(standards)
            error_message += '.'
        raise wtforms.ValidationError(word(error_message))

the_db_adapter = SQLAlchemyAdapter(db, UserModel, UserAuthClass=UserAuthModel, UserInvitationClass=MyUserInvitation)
the_user_manager = UserManager()
the_user_manager.init_app(app, db_adapter=the_db_adapter, login_form=MySignInForm, register_form=MyRegisterForm, user_profile_view_function=user_profile_page, logout_view_function=logout, unauthorized_view_function=unauthorized, unauthenticated_view_function=unauthenticated, login_view_function=custom_login, register_view_function=custom_register, resend_confirm_email_view_function=custom_resend_confirm_email, resend_confirm_email_form=MyResendConfirmEmailForm, password_validator=password_validator, make_safe_url_function=make_safe_url)
lm = LoginManager()
lm.init_app(app)
lm.login_view = 'custom_login'

def login_as_admin(url, url_root):
    found = False
    for admin_user in db.session.execute(select(UserModel).filter_by(nickname='admin').order_by(UserModel.id)).scalars():
        if not found:
            found = True
            current_app.login_manager._update_request_context_with_user(admin_user)
            docassemble.base.functions.this_thread.current_info = dict(user=dict(is_anonymous=False, is_authenticated=True, email=admin_user.email, theid=admin_user.id, the_user_id=admin_user.id, roles=['admin'], firstname=admin_user.first_name, lastname=admin_user.last_name, nickname=admin_user.nickname, country=admin_user.country, subdivisionfirst=admin_user.subdivisionfirst, subdivisionsecond=admin_user.subdivisionsecond, subdivisionthird=admin_user.subdivisionthird, organization=admin_user.organization, location=None, session_uid='admin', device_id='admin'), session=None, secret=None, yaml_filename=final_default_yaml_filename, url=url, url_root=url_root, encrypted=False, action=None, interface='initialization', arguments={})

def import_necessary(url, url_root):
    login_as_admin(url, url_root)
    modules_to_import = daconfig.get('preloaded modules', None)
    if isinstance(modules_to_import, list):
        for module_name in daconfig['preloaded modules']:
            try:
                importlib.import_module(module_name)
            except:
                pass

    start_dir = len(FULL_PACKAGE_DIRECTORY.split(os.sep))
    avoid_dirs = [os.path.join(FULL_PACKAGE_DIRECTORY, 'docassemble', 'base'),
                  os.path.join(FULL_PACKAGE_DIRECTORY, 'docassemble', 'demo'),
                  os.path.join(FULL_PACKAGE_DIRECTORY, 'docassemble', 'webapp')]
    modules = ['docassemble.base.legal']
    for root, dirs, files in os.walk(os.path.join(FULL_PACKAGE_DIRECTORY, 'docassemble')):
        ok = True
        for avoid in avoid_dirs:
            if root.startswith(avoid):
                ok = False
                break
        if not ok:
            continue
        for the_file in files:
            if not the_file.endswith('.py'):
                continue
            thefilename = os.path.join(root, the_file)
            with open(thefilename, 'r', encoding='utf-8') as fp:
                for cnt, line in enumerate(fp):
                    if line.startswith('# do not pre-load'):
                        break
                    if line.startswith('class') or line.startswith('# pre-load') or 'docassemble.base.util.update' in line:
                        parts = thefilename.split(os.sep)[start_dir:]
                        parts[-1] = parts[-1][0:-3]
                        modules.append(('.'.join(parts)))
                        break
    for module_name in modules:
        current_package = re.sub(r'\.[^\.]+$', '', module_name)
        docassemble.base.functions.this_thread.current_package = current_package
        docassemble.base.functions.this_thread.current_info.update(dict(yaml_filename=current_package + ':data/questions/test.yml'))
        try:
            importlib.import_module(module_name)
        except Exception as err:
            sys.stderr.write("Import of " + module_name + " failed.  " + err.__class__.__name__ + ": " + str(err) + "\n")
    current_app.login_manager._update_request_context_with_user()

def get_clicksend_config():
    if 'clicksend' in daconfig and isinstance(daconfig['clicksend'], (list, dict)):
        the_clicksend_config = {'name': {}, 'number': {}}
        if isinstance(daconfig['clicksend'], dict):
            config_list = [daconfig['clicksend']]
        else:
            config_list = daconfig['clicksend']
        for the_config in config_list:
            if isinstance(the_config, dict) and 'api username' in the_config and 'api key' in the_config and 'number' in the_config:
                if 'country' not in the_config:
                    the_config['country'] = docassemble.webapp.backend.DEFAULT_COUNTRY or 'US'
                if 'from email' not in the_config:
                    the_config['from email'] = app.config['MAIL_DEFAULT_SENDER']
                the_clicksend_config['number'][str(the_config['number'])] = the_config
                if 'default' not in the_clicksend_config['name']:
                    the_clicksend_config['name']['default'] = the_config
                if 'name' in the_config:
                    the_clicksend_config['name'][the_config['name']] = the_config
            else:
                sys.stderr.write("improper setup in twilio configuration\n")
        if 'default' not in the_clicksend_config['name']:
            the_clicksend_config = None
    else:
        the_clicksend_config = None
    return the_clicksend_config

clicksend_config = get_clicksend_config()

def get_twilio_config():
    if 'twilio' in daconfig:
        the_twilio_config = {}
        the_twilio_config['account sid'] = {}
        the_twilio_config['number'] = {}
        the_twilio_config['name'] = {}
        if not isinstance(daconfig['twilio'], list):
            config_list = [daconfig['twilio']]
        else:
            config_list = daconfig['twilio']
        for tconfig in config_list:
            if isinstance(tconfig, dict) and 'account sid' in tconfig and 'number' in tconfig:
                the_twilio_config['account sid'][str(tconfig['account sid'])] = 1
                the_twilio_config['number'][str(tconfig['number'])] = tconfig
                if 'default' not in the_twilio_config['name']:
                    the_twilio_config['name']['default'] = tconfig
                if 'name' in tconfig:
                    the_twilio_config['name'][tconfig['name']] = tconfig
            else:
                sys.stderr.write("improper setup in twilio configuration\n")
        if 'default' not in the_twilio_config['name']:
            the_twilio_config = None
    else:
        the_twilio_config = None
    return the_twilio_config

twilio_config = get_twilio_config()

app.debug = False
app.handle_url_build_error = my_default_url
app.config['CONTAINER_CLASS'] = 'container-fluid' if daconfig.get('admin full width', False) else 'container'
app.config['USE_GOOGLE_LOGIN'] = False
app.config['USE_FACEBOOK_LOGIN'] = False
app.config['USE_TWITTER_LOGIN'] = False
app.config['USE_AUTH0_LOGIN'] = False
app.config['USE_KEYCLOAK_LOGIN'] = False
app.config['USE_AZURE_LOGIN'] = False
app.config['USE_GOOGLE_DRIVE'] = False
app.config['USE_ONEDRIVE'] = False
app.config['USE_PHONE_LOGIN'] = False
app.config['USE_GITHUB'] = False
app.config['USE_PASSWORD_LOGIN'] = not bool(daconfig.get('password login', True) is False)
if twilio_config is not None and daconfig.get('phone login', False) is True:
    app.config['USE_PHONE_LOGIN'] = True
if 'oauth' in daconfig:
    app.config['OAUTH_CREDENTIALS'] = daconfig['oauth']
    app.config['USE_GOOGLE_LOGIN'] = bool('google' in daconfig['oauth'] and not ('enable' in daconfig['oauth']['google'] and daconfig['oauth']['google']['enable'] is False))
    app.config['USE_FACEBOOK_LOGIN'] = bool('facebook' in daconfig['oauth'] and not ('enable' in daconfig['oauth']['facebook'] and daconfig['oauth']['facebook']['enable'] is False))
    app.config['USE_TWITTER_LOGIN'] = bool('twitter' in daconfig['oauth'] and not ('enable' in daconfig['oauth']['twitter'] and daconfig['oauth']['twitter']['enable'] is False))
    app.config['USE_AUTH0_LOGIN'] = bool('auth0' in daconfig['oauth'] and not ('enable' in daconfig['oauth']['auth0'] and daconfig['oauth']['auth0']['enable'] is False))
    app.config['USE_KEYCLOAK_LOGIN'] = bool('keycloak' in daconfig['oauth'] and not ('enable' in daconfig['oauth']['keycloak'] and daconfig['oauth']['keycloak']['enable'] is False))
    app.config['USE_AZURE_LOGIN'] = bool('azure' in daconfig['oauth'] and not ('enable' in daconfig['oauth']['azure'] and daconfig['oauth']['azure']['enable'] is False))
    app.config['USE_GOOGLE_DRIVE'] = bool('googledrive' in daconfig['oauth'] and not ('enable' in daconfig['oauth']['googledrive'] and daconfig['oauth']['googledrive']['enable'] is False))
    app.config['USE_ONEDRIVE'] = bool('onedrive' in daconfig['oauth'] and not ('enable' in daconfig['oauth']['onedrive'] and daconfig['oauth']['onedrive']['enable'] is False))
    app.config['USE_GITHUB'] = bool('github' in daconfig['oauth'] and not ('enable' in daconfig['oauth']['github'] and daconfig['oauth']['github']['enable'] is False))
else:
    app.config['OAUTH_CREDENTIALS'] = {}
app.config['USE_PYPI'] = daconfig.get('pypi', False)

if daconfig.get('button size', 'medium') == 'medium':
    app.config['BUTTON_CLASS'] = 'btn-da'
elif daconfig['button size'] == 'large':
    app.config['BUTTON_CLASS'] = 'btn-lg btn-da'
elif daconfig['button size'] == 'small':
    app.config['BUTTON_CLASS'] = 'btn-sm btn-da'
else:
    app.config['BUTTON_CLASS'] = 'btn-da'

if daconfig.get('button style', 'normal') == 'normal':
    app.config['BUTTON_STYLE'] = 'btn-'
elif daconfig['button style'] == 'outline':
    app.config['BUTTON_STYLE'] = 'btn-outline-'
else:
    app.config['BUTTON_STYLE'] = 'btn-'
BUTTON_COLOR_NAV_LOGIN = daconfig['button colors'].get('navigation bar login', 'primary')
app.config['FOOTER_CLASS'] = str(daconfig.get('footer css class', 'bg-light')).strip() + ' dafooter'

def get_page_parts():
    the_page_parts = {}
    if 'global footer' in daconfig:
        if isinstance(daconfig['global footer'], dict):
            the_page_parts['global footer'] = {}
            for lang, val in daconfig['global footer'].items():
                the_page_parts['global footer'][lang] = Markup(val)
        else:
            the_page_parts['global footer'] = {'*': Markup(str(daconfig['global footer']))}

    for page_key in ('login page', 'register page', 'interview page', 'start page', 'profile page', 'reset password page', 'forgot password page', 'change password page', '404 page'):
        for part_key in ('title', 'tab title', 'extra css', 'extra javascript', 'heading', 'pre', 'submit', 'post', 'footer'):
            key = page_key + ' ' + part_key
            if key in daconfig:
                if isinstance(daconfig[key], dict):
                    the_page_parts[key] = {}
                    for lang, val in daconfig[key].items():
                        the_page_parts[key][lang] = Markup(val)
                else:
                    the_page_parts[key] = {'*': Markup(str(daconfig[key]))}

    the_main_page_parts = {}
    lang_list = set()
    main_page_parts_list = (
        'main page back button label',
        'main page continue button label',
        'main page corner back button label',
        'main page exit label',
        'main page exit link',
        'main page exit url',
        'main page footer',
        'main page help label',
        'main page logo',
        'main page navigation bar html',
        'main page post',
        'main page pre',
        'main page resume button label',
        'main page right',
        'main page short logo',
        'main page short title',
        'main page submit',
        'main page subtitle',
        'main page title url opens in other window',
        'main page title url',
        'main page title',
        'main page under')
    for key in main_page_parts_list:
        if key in daconfig and isinstance(daconfig[key], dict):
            for lang in daconfig[key]:
                lang_list.add(lang)
    lang_list.add(DEFAULT_LANGUAGE)
    lang_list.add('*')
    for lang in lang_list:
        the_main_page_parts[lang] = {}
    for key in main_page_parts_list:
        for lang in lang_list:
            if key in daconfig:
                if isinstance(daconfig[key], dict):
                    the_main_page_parts[lang][key] = daconfig[key].get(lang, daconfig[key].get('*', ''))
                else:
                    the_main_page_parts[lang][key] = daconfig[key]
            else:
                the_main_page_parts[lang][key] = ''
        if the_main_page_parts[DEFAULT_LANGUAGE][key] == '' and the_main_page_parts['*'][key] != '':
            the_main_page_parts[DEFAULT_LANGUAGE][key] = the_main_page_parts['*'][key]
    return (the_page_parts, the_main_page_parts)

(page_parts, main_page_parts) = get_page_parts()

ga_configured = bool(google_config.get('analytics id', None) is not None)

if google_config.get('analytics id', None) is not None or daconfig.get('segment id', None) is not None:
    analytics_configured = True
    reserved_argnames = ('i', 'json', 'js_target', 'from_list', 'session', 'cache', 'reset', 'new_session', 'action', 'utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content')
else:
    analytics_configured = False
    reserved_argnames = ('i', 'json', 'js_target', 'from_list', 'session', 'cache', 'reset', 'new_session', 'action')

def get_sms_session(phone_number, config='default'):
    sess_info = None
    if twilio_config is None:
        raise DAError("get_sms_session: Twilio not enabled")
    if config not in twilio_config['name']:
        raise DAError("get_sms_session: Invalid twilio configuration")
    tconfig = twilio_config['name'][config]
    phone_number = docassemble.base.functions.phone_number_in_e164(phone_number)
    if phone_number is None:
        raise DAError("terminate_sms_session: phone_number " + str(phone_number) + " is invalid")
    sess_contents = r.get('da:sms:client:' + phone_number + ':server:' + tconfig['number'])
    if sess_contents is not None:
        try:
            sess_info = fix_pickle_obj(sess_contents)
        except:
            logmessage("get_sms_session: unable to decode session information")
    sess_info['email'] = None
    if 'user_id' in sess_info and sess_info['user_id'] is not None:
        user = load_user(sess_info['user_id'])
        if user is not None:
            sess_info['email'] = user.email
    return sess_info

def initiate_sms_session(phone_number, yaml_filename=None, uid=None, secret=None, encrypted=None, user_id=None, email=None, new=False, config='default'):
    phone_number = docassemble.base.functions.phone_number_in_e164(phone_number)
    if phone_number is None:
        raise DAError("initiate_sms_session: phone_number " + str(phone_number) + " is invalid")
    if config not in twilio_config['name']:
        raise DAError("get_sms_session: Invalid twilio configuration")
    tconfig = twilio_config['name'][config]
    the_current_info = docassemble.base.functions.get_current_info()
    if yaml_filename is None:
        yaml_filename = the_current_info.get('yaml_filename', None)
        if yaml_filename is None:
            yaml_filename = default_yaml_filename
    temp_user_id = None
    if user_id is None and email is not None:
        user = db.session.execute(select(UserModel).where(and_(UserModel.email.ilike(email), UserModel.active == True))).scalar()
        if user is not None:
            user_id = user.id
    if user_id is None:
        if not new:
            if 'user' in the_current_info:
                if 'theid' in the_current_info['user']:
                    if the_current_info['user'].get('is_authenticated', False):
                        user_id = the_current_info['user']['theid']
                    else:
                        temp_user_id = the_current_info['user']['theid']
        if user_id is None and temp_user_id is None:
            new_temp_user = TempUser()
            db.session.add(new_temp_user)
            db.session.commit()
            temp_user_id = new_temp_user.id
    if secret is None:
        if not new:
            secret = the_current_info['secret']
        if secret is None:
            secret = random_string(16)
    if uid is None:
        if new:
            uid = get_unique_name(yaml_filename, secret)
        else:
            uid = the_current_info.get('session', None)
            if uid is None:
                uid = get_unique_name(yaml_filename, secret)
    if encrypted is None:
        if new:
            encrypted = True
        else:
            encrypted = the_current_info['encrypted']
    sess_info = dict(yaml_filename=yaml_filename, uid=uid, secret=secret, number=phone_number, encrypted=encrypted, tempuser=temp_user_id, user_id=user_id)
    #logmessage("initiate_sms_session: setting da:sms:client:" + phone_number + ':server:' + tconfig['number'] + " to " + str(sess_info))
    r.set('da:sms:client:' + phone_number + ':server:' + tconfig['number'], pickle.dumps(sess_info))
    return True

def terminate_sms_session(phone_number, config='default'):
    if config not in twilio_config['name']:
        raise DAError("get_sms_session: Invalid twilio configuration")
    tconfig = twilio_config['name'][config]
    phone_number = docassemble.base.functions.phone_number_in_e164(phone_number)
    r.delete('da:sms:client:' + phone_number + ':server:' + tconfig['number'])

def fix_http(url):
    if HTTP_TO_HTTPS:
        return re.sub(r'^http:', 'https:', url)
    return url

def safe_quote_func(string, safe='', encoding=None, errors=None):
    return urllibquote(string, safe='', encoding=encoding, errors=errors)

def remove_question_package(args):
    if '_question' in args:
        del args['_question']
    if '_package' in args:
        del args['_package']

def get_url_from_file_reference(file_reference, **kwargs):
    if 'jsembed' in docassemble.base.functions.this_thread.misc or COOKIELESS_SESSIONS:
        kwargs['_external'] = True
    privileged = kwargs.get('privileged', False)
    if isinstance(file_reference, DAFileList) and len(file_reference.elements) > 0:
        file_reference = file_reference.elements[0]
    elif isinstance(file_reference, DAFileCollection):
        file_reference = file_reference._first_file()
    elif isinstance(file_reference, DAStaticFile):
        return file_reference.url_for(**kwargs)
    if isinstance(file_reference, DAFile) and hasattr(file_reference, 'number'):
        file_number = file_reference.number
        if privileged or can_access_file_number(file_number, uids=get_session_uids()):
            url_properties = {}
            if hasattr(file_reference, 'filename') and len(file_reference.filename) and file_reference.has_specific_filename:
                url_properties['display_filename'] = file_reference.filename
            if hasattr(file_reference, 'extension'):
                url_properties['ext'] = file_reference.extension
            for key, val in kwargs.items():
                url_properties[key] = val
            the_file = SavedFile(file_number)
            if kwargs.get('temporary', False):
                return the_file.temp_url_for(**url_properties)
            return the_file.url_for(**url_properties)
    file_reference = str(file_reference)
    if re.search(r'^https?://', file_reference) or re.search(r'^mailto:', file_reference) or file_reference.startswith('/') or file_reference.startswith('?'):
        if '?' not in file_reference:
            args = {}
            for key, val in kwargs.items():
                if key in ('_package', '_question', '_external'):
                    continue
                args[key] = val
            if len(args) > 0:
                if file_reference.startswith('mailto:') and 'body' in args:
                    args['body'] = re.sub(r'(?<!\r)\n', '\r\n', args['body'], re.MULTILINE)
                return file_reference + '?' + urlencode(args, quote_via=safe_quote_func)
        return file_reference
    kwargs_with_i = copy.copy(kwargs)
    if 'i' not in kwargs_with_i:
        yaml_filename = docassemble.base.functions.this_thread.current_info.get('yaml_filename', None)
        if yaml_filename is not None:
            kwargs_with_i['i'] = yaml_filename
    if file_reference in ('login', 'signin'):
        remove_question_package(kwargs)
        return url_for('user.login', **kwargs)
    if file_reference == 'profile':
        remove_question_package(kwargs)
        return url_for('user_profile_page', **kwargs)
    if file_reference == 'change_password':
        remove_question_package(kwargs)
        return url_for('user.change_password', **kwargs)
    if file_reference == 'register':
        remove_question_package(kwargs)
        return url_for('user.register', **kwargs)
    if file_reference == 'config':
        remove_question_package(kwargs)
        return url_for('config_page', **kwargs)
    if file_reference == 'leave':
        remove_question_package(kwargs)
        return url_for('leave', **kwargs)
    if file_reference == 'logout':
        remove_question_package(kwargs)
        return url_for('user.logout', **kwargs)
    if file_reference == 'restart':
        remove_question_package(kwargs_with_i)
        return url_for('restart_session', **kwargs_with_i)
    if file_reference == 'new_session':
        remove_question_package(kwargs_with_i)
        return url_for('new_session', **kwargs_with_i)
    if file_reference == 'help':
        return 'javascript:daShowHelpTab()'
    if file_reference == 'interview':
        remove_question_package(kwargs)
        return url_for('index', **kwargs)
    if file_reference == 'flex_interview':
        remove_question_package(kwargs)
        how_called = docassemble.base.functions.this_thread.misc.get('call', None)
        if how_called is None:
            return url_for('index', **kwargs)
        try:
            if int(kwargs.get('new_session')):
                is_new = True
                del kwargs['new_session']
            else:
                is_new = False
        except:
            is_new = False
        if how_called[0] in ('start', 'run'):
            del kwargs['i']
            kwargs['package'] = how_called[1]
            kwargs['filename'] = how_called[2]
            if is_new:
                return url_for('redirect_to_interview_in_package', **kwargs)
            return url_for('run_interview_in_package', **kwargs)
        if how_called[0] in ('start_dispatch', 'run_dispatch'):
            del kwargs['i']
            kwargs['dispatch'] = how_called[1]
            if is_new:
                return url_for('redirect_to_interview', **kwargs)
            return url_for('run_interview', **kwargs)
        if how_called[0] in ('start_directory', 'run_directory'):
            del kwargs['i']
            kwargs['package'] = how_called[1]
            kwargs['directory'] = how_called[2]
            kwargs['filename'] = how_called[3]
            if is_new:
                return url_for('redirect_to_interview_in_package_directory', **kwargs)
            return url_for('run_interview_in_package_directory', **kwargs)
        if is_new:
            kwargs['new_session'] = 1
        return url_for('index', **kwargs)
    if file_reference == 'interviews':
        remove_question_package(kwargs)
        return url_for('interview_list', **kwargs)
    if file_reference == 'exit':
        remove_question_package(kwargs_with_i)
        return url_for('exit_endpoint', **kwargs_with_i)
    if file_reference == 'exit_logout':
        remove_question_package(kwargs_with_i)
        return url_for('exit_logout', **kwargs_with_i)
    if file_reference == 'dispatch':
        remove_question_package(kwargs)
        return url_for('interview_start', **kwargs)
    if file_reference == 'manage':
        remove_question_package(kwargs)
        return url_for('manage_account', **kwargs)
    if file_reference == 'interview_list':
        remove_question_package(kwargs)
        return url_for('interview_list', **kwargs)
    if file_reference == 'playground':
        remove_question_package(kwargs)
        return url_for('playground_page', **kwargs)
    if file_reference == 'playgroundtemplate':
        kwargs['section'] = 'template'
        remove_question_package(kwargs)
        return url_for('playground_files', **kwargs)
    if file_reference == 'playgroundstatic':
        kwargs['section'] = 'static'
        remove_question_package(kwargs)
        return url_for('playground_files', **kwargs)
    if file_reference == 'playgroundsources':
        kwargs['section'] = 'sources'
        remove_question_package(kwargs)
        return url_for('playground_files', **kwargs)
    if file_reference == 'playgroundmodules':
        kwargs['section'] = 'modules'
        remove_question_package(kwargs)
        return url_for('playground_files', **kwargs)
    if file_reference == 'playgroundpackages':
        remove_question_package(kwargs)
        return url_for('playground_packages', **kwargs)
    if file_reference == 'playgroundfiles':
        remove_question_package(kwargs)
        return url_for('playground_files', **kwargs)
    if file_reference == 'create_playground_package':
        remove_question_package(kwargs)
        return url_for('create_playground_package', **kwargs)
    if file_reference == 'configuration':
        remove_question_package(kwargs)
        return url_for('config_page', **kwargs)
    if file_reference == 'root':
        remove_question_package(kwargs)
        return url_for('rootindex', **kwargs)
    if file_reference == 'run':
        remove_question_package(kwargs)
        return url_for('run_interview_in_package', **kwargs)
    if file_reference == 'run_dispatch':
        remove_question_package(kwargs)
        return url_for('run_interview', **kwargs)
    if file_reference == 'run_new':
        remove_question_package(kwargs)
        return url_for('redirect_to_interview_in_package', **kwargs)
    if file_reference == 'run_new_dispatch':
        remove_question_package(kwargs)
        return url_for('redirect_to_interview', **kwargs)
    if re.search('^[0-9]+$', file_reference):
        remove_question_package(kwargs)
        file_number = file_reference
        if kwargs.get('temporary', False):
            url = SavedFile(file_number).temp_url_for(**kwargs)
        elif can_access_file_number(file_number, uids=get_session_uids()):
            url = SavedFile(file_number).url_for(**kwargs)
        else:
            logmessage("Problem accessing " + str(file_number))
            url = 'about:blank'
    else:
        question = kwargs.get('_question', None)
        package_arg = kwargs.get('_package', None)
        if 'ext' in kwargs and kwargs['ext'] is not None:
            extn = kwargs['ext']
            extn = re.sub(r'^\.', '', extn)
            extn = '.' + extn
        else:
            extn = ''
        parts = file_reference.split(':')
        if len(parts) < 2:
            file_reference = re.sub(r'^data/static/', '', file_reference)
            the_package = None
            if question is not None and question.from_source is not None and hasattr(question.from_source, 'package'):
                the_package = question.from_source.package
            if the_package is None and package_arg is not None:
                the_package = package_arg
            if the_package is None:
                the_package = 'docassemble.base'
            parts = [the_package, file_reference]
        parts[1] = re.sub(r'^data/[^/]+/', '', parts[1])
        url = url_if_exists(parts[0] + ':data/static/' + parts[1] + extn, **kwargs)
    return url

def user_id_dict():
    output = {}
    for user in db.session.execute(select(UserModel).options(db.joinedload(UserModel.roles))).unique().scalars():
        output[user.id] = user
    anon = FakeUser()
    anon_role = FakeRole()
    anon_role.name = 'anonymous'
    anon.roles = [anon_role]
    anon.id = -1
    anon.firstname = 'Anonymous'
    anon.lastname = 'User'
    output[-1] = anon
    return output

def get_base_words():
    documentation = get_info_from_file_reference('docassemble.base:data/sources/base-words.yml')
    if 'fullpath' in documentation and documentation['fullpath'] is not None:
        with open(documentation['fullpath'], 'r', encoding='utf-8') as fp:
            content = fp.read()
            content = fix_tabs.sub('  ', content)
            return ruamel.yaml.safe_load(content)
    return None

def get_documentation_dict():
    documentation = get_info_from_file_reference('docassemble.base:data/questions/documentation.yml')
    if 'fullpath' in documentation and documentation['fullpath'] is not None:
        with open(documentation['fullpath'], 'r', encoding='utf-8') as fp:
            content = fp.read()
            content = fix_tabs.sub('  ', content)
            return ruamel.yaml.safe_load(content)
    return None

def get_name_info():
    docstring = get_info_from_file_reference('docassemble.base:data/questions/docstring.yml')
    if 'fullpath' in docstring and docstring['fullpath'] is not None:
        with open(docstring['fullpath'], 'r', encoding='utf-8') as fp:
            content = fp.read()
            content = fix_tabs.sub('  ', content)
            info = ruamel.yaml.safe_load(content)
        for val in info:
            info[val]['name'] = val
            if 'insert' not in info[val]:
                info[val]['insert'] = val
            if 'show' not in info[val]:
                info[val]['show'] = False
            if 'exclude' not in info[val]:
                info[val]['exclude'] = False
        return info
    return None

def get_title_documentation():
    documentation = get_info_from_file_reference('docassemble.base:data/questions/title_documentation.yml')
    if 'fullpath' in documentation and documentation['fullpath'] is not None:
        with open(documentation['fullpath'], 'r', encoding='utf-8') as fp:
            content = fp.read()
            content = fix_tabs.sub('  ', content)
            return ruamel.yaml.safe_load(content)
    return None

def _call_or_get(function_or_property):
    return function_or_property() if callable(function_or_property) else function_or_property

def pad_to_16(the_string):
    if len(the_string) >= 16:
        return the_string[:16]
    return str(the_string) + (16 - len(the_string)) * '0'

def decrypt_session(secret, user_code=None, filename=None):
    #logmessage("decrypt_session: user_code is " + str(user_code) + " and filename is " + str(filename))
    nowtime = datetime.datetime.utcnow()
    if user_code is None or filename is None or secret is None:
        return
    for record in db.session.execute(select(SpeakList).filter_by(key=user_code, filename=filename, encrypted=True).with_for_update()).scalars():
        phrase = decrypt_phrase(record.phrase, secret)
        record.phrase = pack_phrase(phrase)
        record.encrypted = False
    db.session.commit()
    for record in db.session.execute(select(UserDict).filter_by(key=user_code, filename=filename, encrypted=True).order_by(UserDict.indexno).with_for_update()).scalars():
        the_dict = decrypt_dictionary(record.dictionary, secret)
        record.dictionary = pack_dictionary(the_dict)
        record.encrypted = False
        record.modtime = nowtime
    db.session.commit()
    for record in db.session.execute(select(ChatLog).filter_by(key=user_code, filename=filename, encrypted=True).with_for_update()).scalars():
        phrase = decrypt_phrase(record.message, secret)
        record.message = pack_phrase(phrase)
        record.encrypted = False
    db.session.commit()

def encrypt_session(secret, user_code=None, filename=None):
    #logmessage("encrypt_session: user_code is " + str(user_code) + " and filename is " + str(filename))
    nowtime = datetime.datetime.utcnow()
    if user_code is None or filename is None or secret is None:
        return
    for record in db.session.execute(select(SpeakList).filter_by(key=user_code, filename=filename, encrypted=False).with_for_update()).scalars():
        phrase = unpack_phrase(record.phrase)
        record.phrase = encrypt_phrase(phrase, secret)
        record.encrypted = True
    db.session.commit()
    for record in db.session.execute(select(UserDict).filter_by(key=user_code, filename=filename, encrypted=False).order_by(UserDict.indexno).with_for_update()).scalars():
        the_dict = unpack_dictionary(record.dictionary)
        record.dictionary = encrypt_dictionary(the_dict, secret)
        record.encrypted = True
        record.modtime = nowtime
    db.session.commit()
    for record in db.session.execute(select(ChatLog).filter_by(key=user_code, filename=filename, encrypted=False).with_for_update()).scalars():
        phrase = unpack_phrase(record.message)
        record.message = encrypt_phrase(phrase, secret)
        record.encrypted = True
    db.session.commit()

def substitute_secret(oldsecret, newsecret, user=None, to_convert=None):
    if user is None:
        user = current_user
    temp_user = session.get('tempuser', None)
    #logmessage("substitute_secret: " + repr(oldsecret) + " and " + repr(newsecret) + " and temp_user is " + repr(temp_user))
    if oldsecret in ('None', newsecret):
        #logmessage("substitute_secret: returning new secret without doing anything")
        return newsecret
    #logmessage("substitute_secret: continuing")
    if temp_user is not None:
        for object_entry in db.session.execute(select(GlobalObjectStorage).filter_by(user_id=user.id, encrypted=True).with_for_update()).scalars():
            try:
                object_entry.value = encrypt_object(decrypt_object(object_entry.value, oldsecret), newsecret)
            except Exception as err:
                logmessage("Failure to change encryption of object " + object_entry.key + ": " + str(err))
        db.session.commit()
    if to_convert is None:
        to_do = set()
        if 'i' in session and 'uid' in session: #TEMPORARY
            get_session(session['i'])
        if 'sessions' in session:
            for filename, info in session['sessions'].items():
                to_do.add((filename, info['uid']))
        for the_record in db.session.execute(select(UserDict.filename, UserDict.key).filter_by(user_id=user.id).group_by(UserDict.filename, UserDict.key)):
            to_do.add((the_record.filename, the_record.key))
        for the_record in db.session.execute(select(UserDictKeys.filename, UserDictKeys.key).join(UserDict, and_(UserDictKeys.filename == UserDict.filename, UserDictKeys.key == UserDict.key)).where(and_(UserDictKeys.user_id == user.id)).group_by(UserDictKeys.filename, UserDictKeys.key)):
            to_do.add((the_record.filename, the_record.key))
    else:
        to_do = set(to_convert)
    for (filename, user_code) in to_do:
        #obtain_lock(user_code, filename)
        #logmessage("substitute_secret: filename is " + str(filename) + " and key is " + str(user_code))
        for record in db.session.execute(select(SpeakList).filter_by(key=user_code, filename=filename, encrypted=True).with_for_update()).scalars():
            try:
                phrase = decrypt_phrase(record.phrase, oldsecret)
                record.phrase = encrypt_phrase(phrase, newsecret)
            except:
                pass
        db.session.commit()
        for object_entry in db.session.execute(select(GlobalObjectStorage).where(and_(GlobalObjectStorage.key.like('da:uid:' + user_code + ':i:' + filename + ':%'), GlobalObjectStorage.encrypted == True)).with_for_update()).scalars():
            try:
                object_entry.value = encrypt_object(decrypt_object(object_entry.value, oldsecret), newsecret)
            except:
                pass
        db.session.commit()
        for record in db.session.execute(select(UserDict).filter_by(key=user_code, filename=filename, encrypted=True).order_by(UserDict.indexno).with_for_update()).scalars():
            #logmessage("substitute_secret: record was encrypted")
            try:
                the_dict = decrypt_dictionary(record.dictionary, oldsecret)
            except Exception:
                logmessage("substitute_secret: error decrypting dictionary for filename " + filename + " and uid " + user_code)
                continue
            if not isinstance(the_dict, dict):
                logmessage("substitute_secret: dictionary was not a dict for filename " + filename + " and uid " + user_code)
                continue
            if temp_user:
                try:
                    old_entry = the_dict['_internal']['user_local']['t' + str(temp_user)]
                    del the_dict['_internal']['user_local']['t' + str(temp_user)]
                    the_dict['_internal']['user_local'][str(user.id)] = old_entry
                except:
                    pass
            record.dictionary = encrypt_dictionary(the_dict, newsecret)
        db.session.commit()
        if temp_user:
            for record in db.session.execute(select(UserDict).filter_by(key=user_code, filename=filename, encrypted=False).order_by(UserDict.indexno).with_for_update()).scalars():
                try:
                    the_dict = unpack_dictionary(record.dictionary)
                except Exception:
                    logmessage("substitute_secret: error unpacking dictionary for filename " + filename + " and uid " + user_code)
                    continue
                if not isinstance(the_dict, dict):
                    logmessage("substitute_secret: dictionary was not a dict for filename " + filename + " and uid " + user_code)
                    continue
                try:
                    old_entry = the_dict['_internal']['user_local']['t' + str(temp_user)]
                    del the_dict['_internal']['user_local']['t' + str(temp_user)]
                    the_dict['_internal']['user_local'][str(user.id)] = old_entry
                except:
                    pass
                record.dictionary = pack_dictionary(the_dict)
            db.session.commit()
        for record in db.session.execute(select(ChatLog).filter_by(key=user_code, filename=filename, encrypted=True).with_for_update()).scalars():
            try:
                phrase = decrypt_phrase(record.message, oldsecret)
            except Exception as e:
                logmessage("substitute_secret: error decrypting phrase for filename " + filename + " and uid " + user_code)
                continue
            record.message = encrypt_phrase(phrase, newsecret)
        db.session.commit()
        #release_lock(user_code, filename)
    return newsecret

def MD5Hash(data=None):
    if data is None:
        data = ''
    h = MD5.new()
    h.update(bytearray(data, encoding='utf-8'))
    return h

def set_request_active(value):
    global request_active
    request_active = value

def syslog_message(message):
    message = re.sub(r'\n', ' ', message)
    if current_user and current_user.is_authenticated and not current_user.is_anonymous:
        the_user = current_user.email
    else:
        the_user = "anonymous"
    if request_active:
        try:
            sys_logger.debug('%s', LOGFORMAT % {'message': message, 'clientip': get_requester_ip(request), 'yamlfile': docassemble.base.functions.this_thread.current_info.get('yaml_filename', 'na'), 'user': the_user, 'session': docassemble.base.functions.this_thread.current_info.get('session', 'na')})
        except Exception as err:
            sys.stderr.write("Error writing log message " + str(message) + "\n")
            try:
                sys.stderr.write("Error was " + err.__class__.__name__ + ": " + str(err) + "\n")
            except:
                pass
    else:
        try:
            sys_logger.debug('%s', LOGFORMAT % {'message': message, 'clientip': 'localhost', 'yamlfile': 'na', 'user': 'na', 'session': 'na'})
        except Exception as err:
            sys.stderr.write("Error writing log message " + str(message) + "\n")
            try:
                sys.stderr.write("Error was " + err.__class__.__name__ + ": " + str(err) + "\n")
            except:
                pass

def syslog_message_with_timestamp(message):
    syslog_message(time.strftime("%Y-%m-%d %H:%M:%S") + " " + message)

def copy_playground_modules():
    root_dir = os.path.join(FULL_PACKAGE_DIRECTORY, 'docassemble')
    for d in os.listdir(root_dir):
        if re.search(r'^playground[0-9]', d) and os.path.isdir(os.path.join(root_dir, d)):
            try:
                shutil.rmtree(os.path.join(root_dir, d))
            except:
                sys.stderr.write("copy_playground_modules: error deleting " + os.path.join(root_dir, d) + "\n")
    devs = set()
    for user in db.session.execute(select(UserModel.id).join(UserRoles, UserModel.id == UserRoles.user_id).join(Role, UserRoles.role_id == Role.id).where(and_(UserModel.active == True, or_(Role.name == 'admin', Role.name == 'developer')))):
        devs.add(user.id)
    for user_id in devs:
        mod_dir = SavedFile(user_id, fix=True, section='playgroundmodules')
        local_dirs = [(os.path.join(FULL_PACKAGE_DIRECTORY, 'docassemble', 'playground' + str(user_id)), mod_dir.directory)]
        for dirname in mod_dir.list_of_dirs():
            local_dirs.append((os.path.join(FULL_PACKAGE_DIRECTORY, 'docassemble', 'playground' + str(user_id) + dirname), os.path.join(mod_dir.directory, dirname)))
        for local_dir, mod_directory in local_dirs:
            if os.path.isdir(local_dir):
                try:
                    shutil.rmtree(local_dir)
                except:
                    sys.stderr.write("copy_playground_modules: error deleting " + local_dir + " before replacing it\n")
            os.makedirs(local_dir, exist_ok=True)
            #sys.stderr.write("Copying " + str(mod_directory) + " to " + str(local_dir) + "\n")
            for f in [f for f in os.listdir(mod_directory) if re.search(r'^[A-Za-z].*\.py$', f)]:
                shutil.copyfile(os.path.join(mod_directory, f), os.path.join(local_dir, f))
            #shutil.copytree(mod_dir.directory, local_dir)
            with open(os.path.join(local_dir, '__init__.py'), 'w', encoding='utf-8') as the_file:
                the_file.write(init_py_file)

def proc_example_list(example_list, package, directory, examples):
    for example in example_list:
        if isinstance(example, dict):
            for key, value in example.items():
                sublist = []
                proc_example_list(value, package, directory, sublist)
                examples.append({'title': str(key), 'list': sublist})
                break
            continue
        result = {}
        result['id'] = example
        result['interview'] = url_for('index', reset=1, i=package + ":data/questions/" + directory + example + ".yml")
        example_file = package + ":data/questions/" + directory + example + '.yml'
        if package == 'docassemble.base':
            result['image'] = url_for('static', filename=directory + example + ".png", v=da_version)
        else:
            result['image'] = url_for('package_static', package=package, filename=example + ".png")
        #logmessage("Giving it " + example_file)
        file_info = get_info_from_file_reference(example_file)
        #logmessage("Got back " + file_info['fullpath'])
        start_block = 1
        end_block = 2
        if 'fullpath' not in file_info or file_info['fullpath'] is None:
            logmessage("proc_example_list: could not find " + example_file)
            continue
        with open(file_info['fullpath'], 'r', encoding='utf-8') as fp:
            content = fp.read()
            content = fix_tabs.sub('  ', content)
            content = fix_initial.sub('', content)
            blocks = list(map(lambda x: x.strip(), document_match.split(content)))
            if len(blocks) > 0:
                has_context = False
                for block in blocks:
                    if re.search(r'metadata:', block):
                        try:
                            the_block = ruamel.yaml.safe_load(block)
                            if isinstance(the_block, dict) and 'metadata' in the_block:
                                the_metadata = the_block['metadata']
                                result['title'] = the_metadata.get('title', the_metadata.get('short title', word('Untitled')))
                                if isinstance(result['title'], dict):
                                    result['title'] = result['title'].get('en', word('Untitled'))
                                result['title'] = result['title'].rstrip()
                                result['documentation'] = the_metadata.get('documentation', None)
                                start_block = int(the_metadata.get('example start', 1))
                                end_block = int(the_metadata.get('example end', start_block)) + 1
                                break
                        except Exception as err:
                            logmessage("proc_example_list: error processing " + example_file + ": " + str(err))
                            continue
                if 'title' not in result:
                    logmessage("proc_example_list: no title in " + example_file)
                    continue
                if re.search(r'metadata:', blocks[0]) and start_block > 0:
                    initial_block = 1
                else:
                    initial_block = 0
                if start_block > initial_block:
                    result['before_html'] = highlight("\n---\n".join(blocks[initial_block:start_block]) + "\n---", YamlLexer(), HtmlFormatter())
                    has_context = True
                else:
                    result['before_html'] = ''
                if len(blocks) > end_block:
                    result['after_html'] = highlight("---\n" + "\n---\n".join(blocks[end_block:len(blocks)]), YamlLexer(), HtmlFormatter())
                    has_context = True
                else:
                    result['after_html'] = ''
                result['source'] = "\n---\n".join(blocks[start_block:end_block])
                result['html'] = highlight(result['source'], YamlLexer(), HtmlFormatter())
                result['has_context'] = has_context
            else:
                logmessage("proc_example_list: no blocks in " + example_file)
                continue
        examples.append(result)

def get_examples():
    examples = []
    file_list = daconfig.get('playground examples', ['docassemble.base:data/questions/example-list.yml'])
    if not isinstance(file_list, list):
        file_list = [file_list]
    for the_file in file_list:
        if not isinstance(the_file, str):
            continue
        example_list_file = get_info_from_file_reference(the_file)
        if 'fullpath' in example_list_file and example_list_file['fullpath'] is not None:
            if 'package' in example_list_file:
                the_package = example_list_file['package']
            else:
                continue
            if the_package == 'docassemble.base':
                the_directory = 'examples/'
            else:
                the_directory = ''
            if os.path.exists(example_list_file['fullpath']):
                try:
                    with open(example_list_file['fullpath'], 'r', encoding='utf-8') as fp:
                        content = fp.read()
                        content = fix_tabs.sub('  ', content)
                        proc_example_list(ruamel.yaml.safe_load(content), the_package, the_directory, examples)
                except Exception as the_err:
                    logmessage("There was an error loading the Playground examples:" + str(the_err))
    #logmessage("Examples: " + str(examples))
    return examples

def add_timestamps(the_dict, manual_user_id=None):
    nowtime = datetime.datetime.utcnow()
    the_dict['_internal']['starttime'] = nowtime
    the_dict['_internal']['modtime'] = nowtime
    if manual_user_id is not None or (current_user and current_user.is_authenticated and not current_user.is_anonymous):
        if manual_user_id is not None:
            the_user_id = manual_user_id
        else:
            the_user_id = current_user.id
        the_dict['_internal']['accesstime'][the_user_id] = nowtime
    else:
        the_dict['_internal']['accesstime'][-1] = nowtime

def fresh_dictionary():
    the_dict = copy.deepcopy(initial_dict)
    add_timestamps(the_dict)
    return the_dict

def manual_checkout(manual_session_id=None, manual_filename=None, user_id=None, delete_session=False, temp_user_id=None):
    if manual_filename is not None:
        yaml_filename = manual_filename
    else:
        yaml_filename = docassemble.base.functions.this_thread.current_info.get('yaml_filename', None)
    if yaml_filename is None:
        return
    if manual_session_id is not None:
        session_id = manual_session_id
    else:
        session_info = get_session(yaml_filename)
        if session_info is not None:
            session_id = session_info['uid']
        else:
            session_id = None
    if session_id is None:
        return
    if user_id is None:
        if temp_user_id is not None:
            the_user_id = 't' + str(temp_user_id)
        else:
            if current_user.is_anonymous:
                the_user_id = 't' + str(session.get('tempuser', None))
            else:
                the_user_id = current_user.id
    else:
        the_user_id = user_id
    if delete_session:
        if not (not current_user.is_anonymous and user_id != current_user.id):
            clear_specific_session(yaml_filename, session_id)
    endpart = ':uid:' + str(session_id) + ':i:' + str(yaml_filename) + ':userid:' + str(the_user_id)
    pipe = r.pipeline()
    pipe.expire('da:session' + endpart, 12)
    pipe.expire('da:html' + endpart, 12)
    pipe.expire('da:interviewsession' + endpart, 12)
    pipe.expire('da:ready' + endpart, 12)
    pipe.expire('da:block' + endpart, 12)
    pipe.execute()
    #r.publish('da:monitor', json.dumps(dict(messagetype='refreshsessions')))
    #logmessage("Done checking out from " + endpart)

def chat_partners_available(session_id, yaml_filename, the_user_id, mode, partner_roles):
    key = 'da:session:uid:' + str(session_id) + ':i:' + str(yaml_filename) + ':userid:' + str(the_user_id)
    peer_ok = bool(mode in ('peer', 'peerhelp'))
    help_ok = bool(mode in ('help', 'peerhelp'))
    potential_partners = set()
    if help_ok and len(partner_roles) and not r.exists('da:block:uid:' + str(session_id) + ':i:' + str(yaml_filename) + ':userid:' + str(the_user_id)):
        chat_session_key = 'da:interviewsession:uid:' + str(session_id) + ':i:' + str(yaml_filename) + ':userid:' + str(the_user_id)
        for role in partner_roles:
            for the_key in r.keys('da:monitor:role:' + role + ':userid:*'):
                user_id = re.sub(r'^.*:userid:', '', the_key.decode())
                potential_partners.add(user_id)
        for the_key in r.keys('da:monitor:chatpartners:*'):
            the_key = the_key.decode()
            user_id = re.sub(r'^.*chatpartners:', '', the_key)
            if user_id not in potential_partners:
                for chat_key in r.hgetall(the_key):
                    if chat_key.decode() == chat_session_key:
                        potential_partners.add(user_id)
    num_peer = 0
    if peer_ok:
        for sess_key in r.keys('da:session:uid:' + str(session_id) + ':i:' + str(yaml_filename) + ':userid:*'):
            if sess_key.decode() != key:
                num_peer += 1
    result = ChatPartners()
    result.peer = num_peer
    result.help = len(potential_partners)
    return result

def do_redirect(url, is_ajax, is_json, js_target):
    if is_ajax:
        return jsonify(action='redirect', url=url, csrf_token=generate_csrf())
    if is_json:
        if re.search(r'\?', url):
            url = url + '&json=1'
        else:
            url = url + '?json=1'
    if js_target and 'js_target=' not in url:
        if re.search(r'\?', url):
            url = url + '&js_target=' + js_target
        else:
            url = url + '?js_target=' + js_target
    return redirect(url)

def do_refresh(is_ajax, yaml_filename):
    if is_ajax:
        return jsonify(action='refresh', csrf_token=generate_csrf())
    return redirect(url_for('index', i=yaml_filename))

def standard_scripts(interview_language=DEFAULT_LANGUAGE, external=False):
    if interview_language in ('ar', 'cs', 'et', 'he', 'ka', 'nl', 'ro', 'th', 'zh', 'az', 'da', 'fa', 'hu', 'kr', 'no', 'ru', 'tr', 'bg', 'de', 'fi', 'id', 'kz', 'pl', 'sk', 'uk', 'ca', 'el', 'fr', 'it', 'sl', 'uz', 'cr', 'es', 'gl', 'ja', 'lt', 'pt', 'sv', 'vi'):
        fileinput_locale = '\n    <script src="' + url_for('static', filename='bootstrap-fileinput/js/locales/' + interview_language + '.js', v=da_version, _external=external) + '"></script>'
    else:
        fileinput_locale = ''
    return '\n    <script src="' + url_for('static', filename='app/bundle.js', v=da_version, _external=external) + '"></script>' + fileinput_locale

def additional_scripts(interview_status, yaml_filename, as_javascript=False):
    scripts = ''
    interview_package = re.sub(r'^docassemble\.', '', re.sub(r':.*', '', yaml_filename))
    interview_filename = re.sub(r'\.ya?ml$', '', re.sub(r'.*[:\/]', '', yaml_filename), re.IGNORECASE)
    if 'google maps api key' in google_config:
        api_key = google_config.get('google maps api key')
    elif 'api key' in google_config:
        api_key = google_config.get('api key')
    else:
        api_key = None
    if ga_configured and interview_status.question.interview.options.get('analytics on', True):
        ga_id = google_config.get('analytics id')
    else:
        ga_id = None
    output_js = ''
    if api_key is not None:
        region = google_config.get('region', None)
        if region is None:
            region = ''
        else:
            region = '&region=' + region
        scripts += "\n" + '    <script src="https://maps.googleapis.com/maps/api/js?key=' + api_key + region + '&libraries=places"></script>'
        if as_javascript:
            output_js += """\
      var daScript = document.createElement('script');
      daScript.src = "https://maps.googleapis.com/maps/api/js?key=""" + api_key + """&libraries=places";
      document.head.appendChild(daScript);
"""
    if ga_id is not None:
        the_js = """\
      window.dataLayer = window.dataLayer || [];
      function gtag(){dataLayer.push(arguments);}
      gtag('js', new Date());
      function daPageview(){
        var idToUse = daQuestionID['id'];
        if (daQuestionID['ga'] != undefined && daQuestionID['ga'] != null){
          idToUse = daQuestionID['ga'];
        }
        if (idToUse != null){
          gtag('config', """ + json.dumps(ga_id) + """, {""" + ("'cookie_flags': 'SameSite=None;Secure', " if app.config['SESSION_COOKIE_SECURE'] else '') + """'page_path': """ + json.dumps(interview_package) + """ + "/" + """ + json.dumps(interview_filename) + """ + "/" + idToUse.replace(/[^A-Za-z0-9]+/g, '_')});
        }
      }
"""
        scripts += """
    <script async src="https://www.googletagmanager.com/gtag/js?id=""" + ga_id + """"></script>
    <script>
""" + the_js + """
    </script>
"""
        if as_javascript:
# Not good to enable this, since most web sites would have Google Analytics already.
#             output_js += """
#       var daScript = document.createElement('script');
#       daScript.src = "https://www.googletagmanager.com/gtag/js?id=""" + ga_id + """";
#       document.head.appendChild(daScript);
# """
            output_js += the_js
    if as_javascript:
        return output_js
    return scripts

def additional_css(interview_status, js_only=False):
    if 'segment id' in daconfig and interview_status.question.interview.options.get('analytics on', True):
        segment_id = daconfig['segment id']
    else:
        segment_id = None
    start_output = ''
    the_js = ''
    if segment_id is not None:
        segment_js = """\
      !function(){var analytics=window.analytics=window.analytics||[];if(!analytics.initialize)if(analytics.invoked)window.console&&console.error&&console.error("Segment snippet included twice.");else{analytics.invoked=!0;analytics.methods=["trackSubmit","trackClick","trackLink","trackForm","pageview","identify","reset","group","track","ready","alias","debug","page","once","off","on"];analytics.factory=function(t){return function(){var e=Array.prototype.slice.call(arguments);e.unshift(t);analytics.push(e);return analytics}};for(var t=0;t<analytics.methods.length;t++){var e=analytics.methods[t];analytics[e]=analytics.factory(e)}analytics.load=function(t,e){var n=document.createElement("script");n.type="text/javascript";n.async=!0;n.src="https://cdn.segment.com/analytics.js/v1/"+t+"/analytics.min.js";var a=document.getElementsByTagName("script")[0];a.parentNode.insertBefore(n,a);analytics._loadOptions=e};analytics.SNIPPET_VERSION="4.1.0";
      analytics.load(""" + json.dumps(segment_id) + """);
      analytics.page();
      }}();
      function daSegmentEvent(){
        var idToUse = daQuestionID['id'];
        useArguments = false;
        if (daQuestionID['segment'] && daQuestionID['segment']['id']){
          idToUse = daQuestionID['segment']['id'];
          if (daQuestionID['segment']['arguments']){
            for (var keyToUse in daQuestionID['segment']['arguments']){
              if (daQuestionID['segment']['arguments'].hasOwnProperty(keyToUse)){
                useArguments = true;
                break;
              }
            }
          }
        }
        if (idToUse != null){
          if (useArguments){
            analytics.track(idToUse.replace(/[^A-Za-z0-9]+/g, '_'), daQuestionID['segment']['arguments']);
          }
          else{
            analytics.track(idToUse.replace(/[^A-Za-z0-9]+/g, '_'));
          }
        }
      }
"""
        start_output += """
    <script>
""" + segment_js + """\
    </script>"""
        the_js += segment_js
    if len(interview_status.extra_css) > 0:
        start_output += '\n' + indent_by("".join(interview_status.extra_css).strip(), 4).rstrip()
    if js_only:
        return the_js
    return start_output

def standard_html_start(interview_language=DEFAULT_LANGUAGE, debug=False, bootstrap_theme=None, external=False, page_title=None, social=None, yaml_filename=None):
    if social is None:
        social = {}
    if page_title is None:
        page_title = app.config['BRAND_NAME']
    if bootstrap_theme is None and app.config['BOOTSTRAP_THEME'] is not None:
        bootstrap_theme = app.config['BOOTSTRAP_THEME']
    if bootstrap_theme is None:
        bootstrap_part = '\n    <link href="' + url_for('static', filename='bootstrap/css/bootstrap.min.css', v=da_version, _external=external) + '" rel="stylesheet">'
    else:
        bootstrap_part = '\n    <link href="' + bootstrap_theme + '" rel="stylesheet">'
    output = '<!DOCTYPE html>\n<html lang="' + interview_language + '">\n  <head>\n    <meta charset="utf-8">\n    <meta name="mobile-web-app-capable" content="yes">\n    <meta name="apple-mobile-web-app-capable" content="yes">\n    <meta http-equiv="X-UA-Compatible" content="IE=edge">\n    <meta name="viewport" content="width=device-width, initial-scale=1">\n    <link rel="shortcut icon" href="' + url_for('favicon', _external=external) + '">\n    <link rel="apple-touch-icon" sizes="180x180" href="' + url_for('apple_touch_icon', _external=external) + '">\n    <link rel="icon" type="image/png" href="' + url_for('favicon_md', _external=external) + '" sizes="32x32">\n    <link rel="icon" type="image/png" href="' + url_for('favicon_sm', _external=external) + '" sizes="16x16">\n    <link rel="manifest" href="' + url_for('favicon_manifest_json', _external=external) + '">\n    <link rel="mask-icon" href="' + url_for('favicon_safari_pinned_tab', _external=external) + '" color="' + daconfig.get('favicon mask color', '#698aa7') + '">\n    <meta name="theme-color" content="' + daconfig.get('favicon theme color', '#83b3dd') + '">\n    <script defer src="' + url_for('static', filename='fontawesome/js/all.min.js', v=da_version, _external=external) + '"></script>' + bootstrap_part + '\n    <link href="' + url_for('static', filename='app/bundle.css', v=da_version, _external=external) + '" rel="stylesheet">'
    if debug:
        output += '\n    <link href="' + url_for('static', filename='app/pygments.min.css', v=da_version, _external=external) + '" rel="stylesheet">'
    page_title = page_title.replace('\n', ' ').replace('"', '&quot;').strip()
    for key, val in social.items():
        if key not in ('twitter', 'og', 'fb'):
            output += '\n    <meta name="' + key + '" content="' + social[key] + '">'
    if 'description' in social:
        output += '\n    <meta itemprop="description" content="' + social['description'] + '">'
    if 'image' in social:
        output += '\n    <meta itemprop="image" content="' + social['image'] + '">'
    if 'name' in social:
        output += '\n    <meta itemprop="name" content="' + social['name'] + '">'
    else:
        output += '\n    <meta itemprop="name" content="' + page_title + '">'
    if 'twitter' in social:
        if 'card' not in social['twitter']:
            output += '\n    <meta name="twitter:card" content="summary">'
        for key, val in social['twitter'].items():
            output += '\n    <meta name="twitter:' + key + '" content="' + val + '">'
        if 'title' not in social['twitter']:
            output += '\n    <meta name="twitter:title" content="' + page_title + '">'
    if 'fb' in social:
        for key, val in social['fb'].items():
            output += '\n    <meta name="fb:' + key + '" content="' + val + '">'
    if 'og' in social and 'image' in social['og']:
        for key, val in social['og'].items():
            output += '\n    <meta name="og:' + key + '" content="' + val + '">'
        if 'title' not in social['og']:
            output += '\n    <meta name="og:title" content="' + page_title + '">'
        if yaml_filename and 'url' not in social['og']:
            output += '\n    <meta name="og:url" content="' + url_for('index', i=yaml_filename, _external=True) + '">'
        if 'site_name' not in social['og']:
            output += '\n    <meta name="og:site_name" content="' + app.config['BRAND_NAME'].replace('\n', ' ').replace('"', '&quot;').strip() + '">'
        if 'locale' not in social['og']:
            output += '\n    <meta name="og:locale" content="' + app.config['OG_LOCALE'] + '">'
        if 'type' not in social['og']:
            output += '\n    <meta name="og:type" content="website">'
    return output

def process_file(saved_file, orig_file, mimetype, extension, initial=True):
    if extension == "gif" and daconfig.get('imagemagick', 'convert') is not None:
        unconverted = tempfile.NamedTemporaryFile(prefix="datemp", suffix=".gif", delete=False)
        converted = tempfile.NamedTemporaryFile(prefix="datemp", suffix=".png", delete=False)
        shutil.move(orig_file, unconverted.name)
        call_array = [daconfig.get('imagemagick', 'convert'), str(unconverted.name), 'png:' + converted.name]
        try:
            result = subprocess.run(call_array, timeout=60, check=False).returncode
        except subprocess.TimeoutExpired:
            logmessage("process_file: convert from gif took too long")
            result = 1
        if result == 0:
            saved_file.copy_from(converted.name, filename=re.sub(r'\.[^\.]+$', '', saved_file.filename) + '.png')
        else:
            logmessage("process_file: error converting from gif to png")
        shutil.move(unconverted.name, saved_file.path)
        saved_file.save()
    elif extension == "jpg" and daconfig.get('imagemagick', 'convert') is not None:
        unrotated = tempfile.NamedTemporaryFile(prefix="datemp", suffix=".jpg", delete=False)
        rotated = tempfile.NamedTemporaryFile(prefix="datemp", suffix=".jpg", delete=False)
        shutil.move(orig_file, unrotated.name)
        call_array = [daconfig.get('imagemagick', 'convert'), str(unrotated.name), '-auto-orient', '-density', '300', 'jpeg:' + rotated.name]
        try:
            result = subprocess.run(call_array, timeout=60, check=False).returncode
        except subprocess.TimeoutExpired:
            logmessage("process_file: convert from jpeg took too long")
            result = 1
        if result == 0:
            saved_file.copy_from(rotated.name)
        else:
            saved_file.copy_from(unrotated.name)
    elif initial:
        shutil.move(orig_file, saved_file.path)
        saved_file.save()
    # if mimetype == 'video/quicktime' and daconfig.get('ffmpeg', 'ffmpeg') is not None:
    #     call_array = [daconfig.get('ffmpeg', 'ffmpeg'), '-i', saved_file.path + '.' + extension, '-vcodec', 'libtheora', '-acodec', 'libvorbis', saved_file.path + '.ogv']
    #     try:
    #         result = subprocess.run(call_array, timeout=120).returncode
    #     except subprocess.TimeoutExpired:
    #         result = 1
    #     call_array = [daconfig.get('ffmpeg', 'ffmpeg'), '-i', saved_file.path + '.' + extension, '-vcodec', 'copy', '-acodec', 'copy', saved_file.path + '.mp4']
    #     try:
    #         result = subprocess.run(call_array, timeout=120).returncode
    #     except subprocess.TimeoutExpired:
    #         result = 1
    # if mimetype == 'video/mp4' and daconfig.get('ffmpeg', 'ffmpeg') is not None:
    #     call_array = [daconfig.get('ffmpeg', 'ffmpeg'), '-i', saved_file.path + '.' + extension, '-vcodec', 'libtheora', '-acodec', 'libvorbis', saved_file.path + '.ogv']
    #     try:
    #         result = subprocess.run(call_array, timeout=120).returncode
    #     except subprocess.TimeoutExpired:
    #         result = 1
    # if mimetype == 'video/ogg' and daconfig.get('ffmpeg', 'ffmpeg') is not None:
    #     call_array = [daconfig.get('ffmpeg', 'ffmpeg'), '-i', saved_file.path + '.' + extension, '-c:v', 'libx264', '-preset', 'veryslow', '-crf', '22', '-c:a', 'libmp3lame', '-qscale:a', '2', '-ac', '2', '-ar', '44100', saved_file.path + '.mp4']
    #     try:
    #         result = subprocess.run(call_array, timeout=120).returncode
    #     except subprocess.TimeoutExpired:
    #         result = 1
    # if mimetype == 'audio/mpeg' and daconfig.get('pacpl', 'pacpl') is not None:
    #     call_array = [daconfig.get('pacpl', 'pacpl'), '-t', 'ogg', saved_file.path + '.' + extension]
    #     try:
    #         result = subprocess.run(call_array, timeout=120).returncode
    #     except subprocess.TimeoutExpired:
    #         result = 1
    if mimetype == 'audio/ogg' and daconfig.get('pacpl', 'pacpl') is not None:
        call_array = [daconfig.get('pacpl', 'pacpl'), '-t', 'mp3', saved_file.path + '.' + extension]
        try:
            result = subprocess.run(call_array, timeout=120, check=False).returncode
        except subprocess.TimeoutExpired:
            result = 1
    if mimetype == 'audio/3gpp' and daconfig.get('ffmpeg', 'ffmpeg') is not None:
        call_array = [daconfig.get('ffmpeg', 'ffmpeg'), '-i', saved_file.path + '.' + extension, saved_file.path + '.ogg']
        try:
            result = subprocess.run(call_array, timeout=120, check=False).returncode
        except subprocess.TimeoutExpired:
            result = 1
        call_array = [daconfig.get('ffmpeg', 'ffmpeg'), '-i', saved_file.path + '.' + extension, saved_file.path + '.mp3']
        try:
            result = subprocess.run(call_array, timeout=120, check=False).returncode
        except subprocess.TimeoutExpired:
            result = 1
    if mimetype in ('audio/x-wav', 'audio/wav') and daconfig.get('pacpl', 'pacpl') is not None:
        call_array = [daconfig.get('pacpl', 'pacpl'), '-t', 'mp3', saved_file.path + '.' + extension]
        try:
            result = subprocess.run(call_array, timeout=120, check=False).returncode
        except subprocess.TimeoutExpired:
            result = 1
        call_array = [daconfig.get('pacpl', 'pacpl'), '-t', 'ogg', saved_file.path + '.' + extension]
        try:
            result = subprocess.run(call_array, timeout=120, check=False).returncode
        except subprocess.TimeoutExpired:
            result = 1
    #if extension == "pdf":
    #    make_image_files(saved_file.path)
    saved_file.finalize()

def sub_temp_user_dict_key(temp_user_id, user_id):
    temp_interviews = []
    for record in db.session.execute(select(UserDictKeys).filter_by(temp_user_id=temp_user_id).with_for_update()).scalars():
        record.temp_user_id = None
        record.user_id = user_id
        temp_interviews.append((record.filename, record.key))
    db.session.commit()
    return temp_interviews

def sub_temp_other(user):
    if 'tempuser' in session:
        for chat_entry in db.session.execute(select(ChatLog).filter_by(temp_user_id=int(session['tempuser'])).with_for_update()).scalars():
            chat_entry.user_id = user.id
            chat_entry.temp_user_id = None
        db.session.commit()
        for chat_entry in db.session.execute(select(ChatLog).filter_by(temp_owner_id=int(session['tempuser'])).with_for_update()).scalars():
            chat_entry.owner_id = user.id
            chat_entry.temp_owner_id = None
        db.session.commit()
        keys_in_use = {}
        for object_entry in db.session.execute(select(GlobalObjectStorage.id, GlobalObjectStorage.key).filter_by(user_id=user.id)).all():
            if object_entry.key.startswith('da:userid:{:d}:'.format(user.id)):
                if object_entry.key not in keys_in_use:
                    keys_in_use[object_entry.key] = []
                keys_in_use[object_entry.key].append(object_entry.id)
        ids_to_delete = []
        for object_entry in db.session.execute(select(GlobalObjectStorage).filter_by(temp_user_id=int(session['tempuser'])).with_for_update()).scalars():
            object_entry.user_id = user.id
            object_entry.temp_user_id = None
            if object_entry.key.startswith('da:userid:t{:d}:'.format(session['tempuser'])):
                new_key = re.sub(r'^da:userid:t{:d}:'.format(session['tempuser']), 'da:userid:{:d}:'.format(user.id), object_entry.key)
                object_entry.key = new_key
                if new_key in keys_in_use:
                    ids_to_delete.extend(keys_in_use[new_key])
            if object_entry.encrypted and 'newsecret' in session:
                try:
                    object_entry.value = encrypt_object(decrypt_object(object_entry.value, str(request.cookies.get('secret', None))), session['newsecret'])
                except Exception as err:
                    logmessage("Failure to change encryption of object " + object_entry.key + ": " + str(err))
        for the_id in ids_to_delete:
            db.session.execute(delete(GlobalObjectStorage).filter_by(id=the_id))
        db.session.commit()
        db.session.execute(update(UploadsUserAuth).where(UploadsUserAuth.temp_user_id == int(session['tempuser'])).values(user_id=user.id, temp_user_id=None))
        db.session.commit()
        del session['tempuser']

def save_user_dict_key(session_id, filename, priors=False, user=None):
    if user is not None:
        user_id = user.id
        is_auth = True
    else:
        if current_user.is_authenticated and not current_user.is_anonymous:
            is_auth = True
            user_id = current_user.id
        else:
            is_auth = False
            user_id = session.get('tempuser', None)
            if user_id is None:
                logmessage("save_user_dict_key: no user ID available for saving")
                return
    #logmessage("save_user_dict_key: called")
    the_interview_list = set([filename])
    found = set()
    if priors:
        for the_record in db.session.execute(select(UserDict.filename).filter_by(key=session_id).group_by(UserDict.filename)):
            the_interview_list.add(the_record.filename)
    for filename_to_search in the_interview_list:
        if is_auth:
            for the_record in db.session.execute(select(UserDictKeys).filter_by(key=session_id, filename=filename_to_search, user_id=user_id)):
                found.add(filename_to_search)
        else:
            for the_record in db.session.execute(select(UserDictKeys).filter_by(key=session_id, filename=filename_to_search, temp_user_id=user_id)):
                found.add(filename_to_search)
    for filename_to_save in (the_interview_list - found):
        if is_auth:
            new_record = UserDictKeys(key=session_id, filename=filename_to_save, user_id=user_id)
        else:
            new_record = UserDictKeys(key=session_id, filename=filename_to_save, temp_user_id=user_id)
        db.session.add(new_record)
        db.session.commit()

def save_user_dict(user_code, user_dict, filename, secret=None, changed=False, encrypt=True, manual_user_id=None, steps=None, max_indexno=None):
    #logmessage("save_user_dict: called with encrypt " + str(encrypt))
    if REQUIRE_IDEMPOTENT:
        for var_name in ('x', 'i', 'j', 'k', 'l', 'm', 'n'):
            if var_name in user_dict:
                del user_dict[var_name]
        user_dict['_internal']['objselections'] = {}
    if 'session_local' in user_dict:
        del user_dict['session_local']
    if 'device_local' in user_dict:
        del user_dict['device_local']
    if 'user_local' in user_dict:
        del user_dict['user_local']
    nowtime = datetime.datetime.utcnow()
    if steps is not None:
        user_dict['_internal']['steps'] = steps
    user_dict['_internal']['modtime'] = nowtime
    if manual_user_id is not None or (current_user and current_user.is_authenticated and not current_user.is_anonymous):
        if manual_user_id is not None:
            the_user_id = manual_user_id
        else:
            the_user_id = current_user.id
        user_dict['_internal']['accesstime'][the_user_id] = nowtime
    else:
        user_dict['_internal']['accesstime'][-1] = nowtime
        the_user_id = None
    if changed is True:
        if encrypt:
            new_record = UserDict(modtime=nowtime, key=user_code, dictionary=encrypt_dictionary(user_dict, secret), filename=filename, user_id=the_user_id, encrypted=True)
        else:
            new_record = UserDict(modtime=nowtime, key=user_code, dictionary=pack_dictionary(user_dict), filename=filename, user_id=the_user_id, encrypted=False)
        db.session.add(new_record)
        db.session.commit()
    else:
        if max_indexno is None:
            max_indexno = db.session.execute(select(db.func.max(UserDict.indexno)).where(and_(UserDict.key == user_code, UserDict.filename == filename))).scalar()
        if max_indexno is None:
            if encrypt:
                new_record = UserDict(modtime=nowtime, key=user_code, dictionary=encrypt_dictionary(user_dict, secret), filename=filename, user_id=the_user_id, encrypted=True)
            else:
                new_record = UserDict(modtime=nowtime, key=user_code, dictionary=pack_dictionary(user_dict), filename=filename, user_id=the_user_id, encrypted=False)
            db.session.add(new_record)
            db.session.commit()
        else:
            for record in db.session.execute(select(UserDict).filter_by(key=user_code, filename=filename, indexno=max_indexno).with_for_update()).scalars():
                if encrypt:
                    record.dictionary = encrypt_dictionary(user_dict, secret)
                    record.modtime = nowtime
                    record.encrypted = True
                else:
                    record.dictionary = pack_dictionary(user_dict)
                    record.modtime = nowtime
                    record.encrypted = False
            db.session.commit()

def process_bracket_expression(match):
    if match.group(1) in ('B', 'R'):
        try:
            inner = codecs.decode(repad(bytearray(match.group(2), encoding='utf-8')), 'base64').decode('utf-8')
        except:
            inner = match.group(2)
    else:
        inner = match.group(2)
    return "[" + repr(inner) + "]"

def myb64unquote(the_string):
    return codecs.decode(repad(bytearray(the_string, encoding='utf-8')), 'base64').decode('utf-8')

def safeid(text):
    return re.sub(r'[\n=]', '', codecs.encode(text.encode('utf-8'), 'base64').decode())

def from_safeid(text):
    return codecs.decode(repad(bytearray(text, encoding='utf-8')), 'base64').decode('utf-8')

def repad(text):
    return text + (equals_byte * ((4 - len(text) % 4) % 4))

def test_for_valid_var(varname):
    if not valid_python_var.match(varname):
        raise DAError(varname + " is not a valid name.  A valid name consists only of letters, numbers, and underscores, and begins with a letter.")

def navigation_bar(nav, interview, wrapper=True, inner_div_class=None, inner_div_extra=None, show_links=None, hide_inactive_subs=True, a_class=None, show_nesting=True, include_arrows=False, always_open=False, return_dict=None):
    if show_links is None:
        show_links = not bool(hasattr(nav, 'disabled') and nav.disabled)
    if inner_div_class is None:
        inner_div_class = 'nav flex-column nav-pills danav danavlinks danav-vertical danavnested'
    if inner_div_extra is None:
        inner_div_extra = ''
    if a_class is None:
        a_class = 'nav-link danavlink'
        muted_class = ' text-muted'
    else:
        muted_class = ''
    #logmessage("navigation_bar: starting: " + str(section))
    the_language = docassemble.base.functions.get_language()
    non_progressive = bool(hasattr(nav, 'progressive') and not nav.progressive)
    auto_open = bool(always_open or (hasattr(nav, 'auto_open') and nav.auto_open))
    if the_language not in nav.sections:
        the_language = DEFAULT_LANGUAGE
    if the_language not in nav.sections:
        the_language = '*'
    if the_language not in nav.sections:
        return ''
        #raise DAError("Could not find a navigation bar to display.  " + str(nav.sections))
    the_sections = nav.sections[the_language]
    if len(the_sections) == 0:
        return ''
    if docassemble.base.functions.this_thread.current_question.section is not None:
        the_section = docassemble.base.functions.this_thread.current_question.section
    else:
        the_section = nav.current
    #logmessage("Current section is " + repr(the_section))
    #logmessage("Past sections are: " + str(nav.past))
    if the_section is None:
        if isinstance(the_sections[0], dict):
            the_section = list(the_sections[0])[0]
        else:
            the_section = the_sections[0]
    if wrapper:
        output = '<div role="navigation" class="offset-xl-1 col-xl-2 col-lg-3 col-md-3 d-none d-md-block danavdiv">\n  <div class="nav flex-column nav-pills danav danav-vertical danavlinks">\n'
    else:
        output = ''
    section_reached = False
    indexno = 0
    seen = set()
    on_first = True
    #logmessage("Sections is " + repr(the_sections))
    for x in the_sections:
        if include_arrows and not on_first:
            output += '<span class="dainlinearrow"><i class="fas fa-chevron-right"></i></span>'
        on_first = False
        indexno += 1
        the_key = None
        subitems = None
        currently_active = False
        if isinstance(x, dict):
            #logmessage("It is a dict")
            if len(x) == 2 and 'subsections' in x:
                for key, val in x.items():
                    if key == 'subsections':
                        subitems = val
                    else:
                        the_key = key
                        test_for_valid_var(the_key)
                        the_title = val
            elif len(x) == 1:
                #logmessage("The len is one")
                the_key = list(x)[0]
                test_for_valid_var(the_key)
                value = x[the_key]
                if isinstance(value, list):
                    subitems = value
                    the_title = the_key
                else:
                    the_title = value
            else:
                raise DAError("navigation_bar: too many keys in dict.  " + str(the_sections))
        else:
            #logmessage("It is not a dict")
            the_key = None
            the_title = str(x)
        if (the_key is not None and the_section == the_key) or the_section == the_title:
            #output += '<li role="presentation" class="' + li_class + ' active">'
            section_reached = True
            currently_active = True
            active_class = ' active'
            if return_dict is not None:
                return_dict['parent_key'] = the_key
                return_dict['parent_title'] = the_title
                return_dict['key'] = the_key
                return_dict['title'] = the_title
        else:
            active_class = ''
            #output += '<li class="' + li_class + '" role="presentation">'
        new_key = the_title if the_key is None else the_key
        seen.add(new_key)
        #logmessage("new_key is: " + str(new_key))
        #logmessage("seen sections are: " + str(seen))
        #logmessage("nav past sections are: " + repr(nav.past))
        relevant_past = nav.past.intersection(set(nav.section_ids()))
        seen_more = bool(len(relevant_past.difference(seen)) > 0 or new_key in nav.past or the_title in nav.past)
        if non_progressive:
            seen_more = True
            section_reached = False
        #logmessage("the title is " + str(the_title) + " and non_progressive is " + str(non_progressive) + " and show links is " + str(show_links) + " and seen_more is " + str(seen_more) + " and active_class is " + repr(active_class) + " and currently_active is " + str(currently_active) + " and section_reached is " + str(section_reached) + " and the_key is " + str(the_key) + " and interview is " + str(interview) + " and in q is " + ('in q' if the_key in interview.questions else 'not in q'))
        if show_links and (seen_more or currently_active or not section_reached) and the_key is not None and interview is not None and the_key in interview.questions:
            #url = docassemble.base.functions.interview_url_action(the_key)
            if section_reached and not currently_active and not seen_more:
                output += '<span tabindex="-1" data-index="' + str(indexno) + '" class="' + a_class + ' danotavailableyet' + muted_class + '">' + str(the_title) + '</span>'
            else:
                if active_class == '' and not (seen_more and not section_reached):
                    output += '<span tabindex="-1" data-index="' + str(indexno) + '" class="' + a_class + ' inactive' + muted_class + '">' + str(the_title) + '</span>'
                else:
                    output += '<a href="#" data-key="' + the_key + '" data-index="' + str(indexno) + '" class="daclickable ' + a_class + active_class + '">' + str(the_title) + '</a>'
        else:
            if section_reached and not currently_active and not seen_more:
                output += '<span tabindex="-1" data-index="' + str(indexno) + '" class="' + a_class + ' danotavailableyet' + muted_class + '">' + str(the_title) + '</span>'
            else:
                if active_class == '' and not (seen_more and not section_reached):
                    output += '<span tabindex="-1" data-index="' + str(indexno) + '" class="' + a_class + ' inactive' + muted_class + '">' + str(the_title) + '</span>'
                else:
                    output += '<a tabindex="-1" data-index="' + str(indexno) + '" class="' + a_class + active_class + '">' + str(the_title) + '</a>'
        suboutput = ''
        if subitems:
            current_is_within = False
            oldindexno = indexno
            for y in subitems:
                if include_arrows:
                    suboutput += '<span class="dainlinearrow"><i class="fas fa-chevron-right"></i></span>'
                indexno += 1
                sub_currently_active = False
                if isinstance(y, dict):
                    if len(y) == 1:
                        sub_key = list(y)[0]
                        test_for_valid_var(sub_key)
                        sub_title = y[sub_key]
                    else:
                        raise DAError("navigation_bar: too many keys in dict.  " + str(the_sections))
                else:
                    sub_key = None
                    sub_title = str(y)
                if (sub_key is not None and the_section == sub_key) or the_section == sub_title:
                    #suboutput += '<li class="' + li_class + ' active" role="presentation">'
                    section_reached = True
                    current_is_within = True
                    sub_currently_active = True
                    sub_active_class = ' active'
                    if return_dict is not None:
                        return_dict['key'] = sub_key
                        return_dict['title'] = sub_title
                else:
                    sub_active_class = ''
                    #suboutput += '<li class="' + li_class + '" role="presentation">'
                new_sub_key = sub_title if sub_key is None else sub_key
                seen.add(new_sub_key)
                #logmessage("sub: seen sections are: " + str(seen))
                relevant_past = nav.past.intersection(set(nav.section_ids()))
                seen_more = bool(len(relevant_past.difference(seen)) > 0 or new_sub_key in nav.past or sub_title in nav.past)
                if non_progressive:
                    #logmessage("Setting seen_more to True bc non-progressive")
                    seen_more = True
                    section_reached = False
                #logmessage("First sub is %s, indexno is %d, sub_currently_active is %s, sub_key is %s, sub_title is %s, section_reached is %s, current_is_within is %s, sub_active_class is %s, new_sub_key is %s, seen_more is %s, section_reached is %s, show_links is %s" % (str(first_sub), indexno, str(sub_currently_active), sub_key, sub_title, section_reached, current_is_within, sub_active_class, new_sub_key, str(seen_more), str(section_reached), str(show_links)))
                if show_links and (seen_more or sub_currently_active or not section_reached) and sub_key is not None and interview is not None and sub_key in interview.questions:
                    #url = docassemble.base.functions.interview_url_action(sub_key)
                    suboutput += '<a href="#" data-key="' + sub_key + '" data-index="' + str(indexno) + '" class="daclickable ' + a_class + sub_active_class + '">' + str(sub_title) + '</a>'
                else:
                    if section_reached and not sub_currently_active and not seen_more:
                        suboutput += '<span tabindex="-1" data-index="' + str(indexno) + '" class="' + a_class + ' danotavailableyet' + muted_class + '">' + str(sub_title) + '</span>'
                    else:
                        suboutput += '<a tabindex="-1" data-index="' + str(indexno) + '" class="' + a_class + sub_active_class + ' inactive">' + str(sub_title) + '</a>'
                #suboutput += "</li>"
            if currently_active or current_is_within or hide_inactive_subs is False or show_nesting:
                if currently_active or current_is_within or auto_open:
                    suboutput = '<div class="' + inner_div_class + '"' + inner_div_extra + '>' + suboutput
                else:
                    suboutput = '<div style="display: none;" class="danotshowing ' + inner_div_class + '"' + inner_div_extra + '>' + suboutput
                suboutput += "</div>"
                output += suboutput
            else:
                indexno = oldindexno
        #output += "</li>"
    if wrapper:
        output += "\n</div>\n</div>\n"
    if (not non_progressive) and (not section_reached):
        logmessage("Section \"" + str(the_section) + "\" did not exist.")
    return output

def progress_bar(progress, interview):
    if progress is None:
        return ''
    progress = float(progress)
    if progress <= 0:
        return ''
    progress = min(progress, 100)
    if hasattr(interview, 'show_progress_bar_percentage') and interview.show_progress_bar_percentage:
        percentage = str(int(progress)) + '%'
    else:
        percentage = ''
    return '<div class="progress mt-2"><div class="progress-bar" role="progressbar" aria-valuenow="' + str(progress) + '" aria-valuemin="0" aria-valuemax="100" style="width: ' + str(progress) + '%;">' + percentage + '</div></div>\n'

def get_unique_name(filename, secret):
    nowtime = datetime.datetime.utcnow()
    while True:
        newname = random_alphanumeric(32)
        obtain_lock(newname, filename)
        existing_key = db.session.execute(select(UserDict).filter_by(key=newname)).first()
        if existing_key:
            release_lock(newname, filename)
            continue
        new_user_dict = UserDict(modtime=nowtime, key=newname, filename=filename, dictionary=encrypt_dictionary(fresh_dictionary(), secret))
        db.session.add(new_user_dict)
        db.session.commit()
        return newname

def obtain_lock(user_code, filename):
    key = 'da:lock:' + user_code + ':' + filename
    #sys.stderr.write("obtain_lock: getting " + key + "\n")
    found = False
    count = 4
    while count > 0:
        record = r.get(key)
        if record:
            sys.stderr.write("obtain_lock: waiting for " + key + "\n")
            time.sleep(1.0)
        else:
            found = False
            break
        found = True
        count -= 1
    if found:
        sys.stderr.write("Request for " + key + " deadlocked\n")
        release_lock(user_code, filename)
    pipe = r.pipeline()
    pipe.set(key, 1)
    pipe.expire(key, 4)
    pipe.execute()

def obtain_lock_patiently(user_code, filename):
    key = 'da:lock:' + user_code + ':' + filename
    #sys.stderr.write("obtain_lock: getting " + key + "\n")
    found = False
    count = 20
    while count > 0:
        record = r.get(key)
        if record:
            sys.stderr.write("obtain_lock: waiting for " + key + "\n")
            time.sleep(3.0)
        else:
            found = False
            break
        found = True
        count -= 1
    if found:
        sys.stderr.write("Request for " + key + " deadlocked\n")
        release_lock(user_code, filename)
    pipe = r.pipeline()
    pipe.set(key, 1)
    pipe.expire(key, 4)
    pipe.execute()

def release_lock(user_code, filename):
    key = 'da:lock:' + user_code + ':' + filename
    #sys.stderr.write("obtain_lock: releasing " + key + "\n")
    r.delete(key)

def make_navbar(status, steps, show_login, chat_info, debug_mode, index_params, extra_class=None):
    if 'inverse navbar' in status.question.interview.options:
        if status.question.interview.options['inverse navbar']:
            inverse = 'navbar-dark bg-dark '
        else:
            inverse = 'navbar-light bg-light '
    elif daconfig.get('inverse navbar', True):
        inverse = 'navbar-dark bg-dark '
    else:
        inverse = 'navbar-light bg-light '
    if 'jsembed' in docassemble.base.functions.this_thread.misc:
        fixed_top = ''
    else:
        fixed_top = ' fixed-top'
    if extra_class is not None:
        fixed_top += ' ' + extra_class
    navbar = """\
    <div class="navbar""" + fixed_top + """ navbar-expand-md """ + inverse + '"' + """ role="banner">
      <div class="container danavcontainer justify-content-start">
"""
    if status.question.can_go_back and steps > 1:
        if status.question.interview.navigation_back_button:
            navbar += """\
        <form style="display: inline-block" id="dabackbutton" method="POST" action=""" + json.dumps(url_for('index', **index_params)) + """><input type="hidden" name="csrf_token" value=""" + '"' + generate_csrf() + '"' + """/><input type="hidden" name="_back_one" value="1"/><button class="navbar-brand navbar-nav dabackicon dabackbuttoncolor me-3" type="submit" title=""" + json.dumps(word("Go back to the previous question")) + """><span class="nav-link"><i class="fas fa-chevron-left"></i><span class="daback">""" + status.cornerback + """</span></span></button></form>
"""
        else:
            navbar += """\
        <form style="display: inline-block" id="dabackbutton" method="POST" action=""" + json.dumps(url_for('index', **index_params)) + """><input type="hidden" name="csrf_token" value=""" + '"' + generate_csrf() + '"' + """/><input type="hidden" name="_back_one" value="1"/></form>
"""
    if status.title_url:
        if str(status.title_url_opens_in_other_window) == 'False':
            target = ''
        else:
            target = ' target="_blank"'
        navbar += """\
        <a id="dapagetitle" class="navbar-brand danavbar-title dapointer" href=""" + '"' + status.title_url + '"' + target + """><span class="d-none d-md-block">""" + status.display_title + """</span><span class="d-block d-md-none">""" + status.display_short_title + """</span></a>
"""
    else:
        navbar += """\
        <span id="dapagetitle" class="navbar-brand danavbar-title"><span class="d-none d-md-block">""" + status.display_title + """</span><span class="d-block d-md-none">""" + status.display_short_title + """</span></span>
"""
    help_message = word("Help is available")
    help_label = None
    if status.question.interview.question_help_button:
        the_sections = status.interviewHelpText
    else:
        the_sections = status.helpText + status.interviewHelpText
    for help_section in the_sections:
        if help_section['label']:
            help_label = help_section['label']
            break
    if help_label is None:
        help_label = status.extras.get('help label text', None)
    if help_label is None:
        help_label = status.question.help()
    extra_help_message = word("Help is available for this question")
    phone_sr = word("Phone help")
    phone_message = word("Phone help is available")
    chat_sr = word("Live chat")
    source_message = word("Information for the developer")
    if debug_mode:
        source_button = '<li class="nav-item d-none d-md-block"><button class="btn btn-link nav-link da-no-outline" title=' + json.dumps(source_message) + ' id="dasourcetoggle" data-bs-toggle="collapse" data-bs-target="#dasource"><i class="fas fa-code"></i></button></li>'
        source_menu_item = '<a class="dropdown-item d-block d-md-none" title=' + json.dumps(source_message) + ' id="dasourcetoggle" href="#dasource" data-bs-toggle="collapse" aria-expanded="false" aria-controls="source">' + word('Source') + '</a>'
    else:
        source_button = ''
        source_menu_item = ''
    navbar += '        <ul class="nav navbar-nav damynavbar-right" role="tablist">' + source_button + '<li class="nav-item dainvisible" role="presentation"><button class="btn btn-link nav-link active da-no-outline" id="daquestionlabel" data-bs-toggle="tab" data-bs-target="#daquestion">' + word('Question') + '</button></li>'
    if len(status.interviewHelpText) > 0 or (len(status.helpText) > 0 and not status.question.interview.question_help_button):
        if status.question.helptext is None or status.question.interview.question_help_button:
            navbar += '<li class="nav-item" role="presentation"><button class="btn btn-link nav-link dahelptrigger da-no-outline" data-bs-target="#dahelp" data-bs-toggle="tab" id="dahelptoggle" title=' + json.dumps(help_message) + '>' + help_label + '</button></li>'
        else:
            navbar += '<li class="nav-item" role="presentation"><button class="btn btn-link nav-link dahelptrigger da-no-outline daactivetext" data-bs-target="#dahelp" data-bs-toggle="tab" id="dahelptoggle" title=' + json.dumps(extra_help_message) + '>' + help_label + ' <i class="fas fa-star"></i></button></li>'
    else:
        navbar += '<li class="nav-item dainvisible" role="presentation"><button class="btn btn-link nav-link dahelptrigger da-no-outline" id="dahelptoggle" data-bs-target="#dahelp" data-bs-toggle="tab">' + word('Help') + '</button></li>'
    navbar += '<li class="nav-item dainvisible" id="daPhoneAvailable"><button data-bs-target="#dahelp" data-bs-toggle="tab" title=' + json.dumps(phone_message) + ' class="btn btn-link nav-link dapointer dahelptrigger da-no-outline"><i class="fas fa-phone da-chat-active"></i><span class="visually-hidden">' + phone_sr + '</span></button></li><li class="nav-item dainvisible" id="daChatAvailable"><button data-bs-target="#dahelp" data-bs-toggle="tab" class="btn btn-link nav-link dapointer dahelptrigger da-no-outline"><i class="fas fa-comment-alt"></i><span class="visually-hidden">' + chat_sr + '</span></button></li></ul>'
    navbar += """
        <button id="damobile-toggler" type="button" class="navbar-toggler ms-auto" data-bs-toggle="collapse" data-bs-target="#danavbar-collapse">
          <span class="navbar-toggler-icon"></span><span class="visually-hidden">""" + word("Display the menu") + """</span>
        </button>
        <div class="collapse navbar-collapse" id="danavbar-collapse">
          <ul class="navbar-nav ms-auto">
"""
    navbar += status.nav_item
    if 'menu_items' in status.extras:
        if not isinstance(status.extras['menu_items'], list):
            custom_menu = '<a tabindex="-1" class="dropdown-item">' + word("Error: menu_items is not a Python list") + '</a>'
        elif len(status.extras['menu_items']) > 0:
            custom_menu = ""
            for menu_item in status.extras['menu_items']:
                if not (isinstance(menu_item, dict) and 'url' in menu_item and 'label' in menu_item):
                    custom_menu += '<a tabindex="-1" class="dropdown-item">' + word("Error: menu item is not a Python dict with keys of url and label") + '</li>'
                else:
                    screen_size = menu_item.get('screen_size', '')
                    if screen_size == 'small':
                        menu_item_classes = ' d-block d-md-none'
                    elif screen_size == 'large':
                        menu_item_classes = ' d-none d-md-block'
                    else:
                        menu_item_classes = ''
                    match_action = re.search(r'^\?action=([^\&]+)', menu_item['url'])
                    if match_action:
                        custom_menu += '<a class="dropdown-item' + menu_item_classes + '" data-embaction="' + match_action.group(1) + '" href="' + menu_item['url'] + '">' + menu_item['label'] + '</a>'
                    else:
                        custom_menu += '<a class="dropdown-item' + menu_item_classes + '" href="' + menu_item['url'] + '">' + menu_item['label'] + '</a>'
        else:
            custom_menu = False
    else:
        custom_menu = False
    if ALLOW_REGISTRATION:
        sign_in_text = word('Sign in or sign up to save answers')
    else:
        sign_in_text = word('Sign in to save answers')
    if daconfig.get('resume interview after login', False):
        login_url = url_for('user.login', next=url_for('index', **index_params))
    else:
        login_url = url_for('user.login')
    if show_login:
        if current_user.is_anonymous:
            if custom_menu:
                navbar += '            <li class="nav-item dropdown"><a href="#" class="nav-link dropdown-toggle d-none d-md-block" data-bs-toggle="dropdown" role="button" id="damenuLabel" aria-haspopup="true" aria-expanded="false">' + word("Menu") + '</a><div class="dropdown-menu dropdown-menu-end" aria-labelledby="damenuLabel">' + custom_menu + '<a class="dropdown-item" href="' + login_url + '">' + sign_in_text + '</a></div></li>'
            else:
                if daconfig.get('login link style', 'normal') == 'button':
                    if ALLOW_REGISTRATION:
                        if daconfig.get('resume interview after login', False):
                            register_url = url_for('user.register', next=url_for('index', **index_params))
                        else:
                            register_url = url_for('user.register')
                        navbar += '            <li class="nav-item"><a class="nav-link" href="' + register_url + '">' + word('Sign up') + '</a></li>'
                        navbar += '            <li class="nav-item"><a class="nav-link d-block d-md-none" href="' + login_url + '">' + word('Sign in') + '</a>'

                else:
                    navbar += '            <li class="nav-item"><a class="nav-link" href="' + login_url + '">' + sign_in_text + '</a></li>'
        else:
            if (custom_menu is False or custom_menu == '') and status.question.interview.options.get('hide standard menu', False):
                navbar += '            <li class="nav-item"><a class="nav-link" tabindex="-1">' + (current_user.email if current_user.email else re.sub(r'.*\$', '', current_user.social_id)) + '</a></li>'
            else:
                navbar += '            <li class="nav-item dropdown"><a class="nav-link dropdown-toggle d-none d-md-block" href="#" data-bs-toggle="dropdown" role="button" id="damenuLabel" aria-haspopup="true" aria-expanded="false">' + (current_user.email if current_user.email else re.sub(r'.*\$', '', current_user.social_id)) + '</a><div class="dropdown-menu dropdown-menu-end" aria-labelledby="damenuLabel">'
                if custom_menu:
                    navbar += custom_menu
                if not status.question.interview.options.get('hide standard menu', False):
                    if current_user.has_role('admin', 'developer'):
                        navbar += source_menu_item
                    if current_user.has_role('admin', 'advocate') and app.config['ENABLE_MONITOR']:
                        navbar += '<a class="dropdown-item" href="' + url_for('monitor') + '">' + word('Monitor') + '</a>'
                    if current_user.has_role('admin', 'developer', 'trainer'):
                        navbar += '<a class="dropdown-item" href="' + url_for('train') + '">' + word('Train') + '</a>'
                    if current_user.has_role('admin', 'developer'):
                        if app.config['ALLOW_UPDATES']:
                            navbar +='<a class="dropdown-item" href="' + url_for('update_package') + '">' + word('Package Management') + '</a>'
                        navbar +='<a class="dropdown-item" href="' + url_for('logs') + '">' + word('Logs') + '</a>'
                        if app.config['ENABLE_PLAYGROUND']:
                            navbar +='<a class="dropdown-item" href="' + url_for('playground_page') + '">' + word('Playground') + '</a>'
                        navbar +='<a class="dropdown-item" href="' + url_for('utilities') + '">' + word('Utilities') + '</a>'
                        if current_user.has_role('admin'):
                            navbar +='<a class="dropdown-item" href="' + url_for('user_list') + '">' + word('User List') + '</a>'
                            navbar +='<a class="dropdown-item" href="' + url_for('config_page') + '">' + word('Configuration') + '</a>'
                    if app.config['SHOW_DISPATCH']:
                        navbar += '<a class="dropdown-item" href="' + url_for('interview_start') + '">' + word('Available Interviews') + '</a>'
                    for item in app.config['ADMIN_INTERVIEWS']:
                        if item.can_use() and docassemble.base.functions.this_thread.current_info.get('yaml_filename', '') != item.interview:
                            navbar += '<a class="dropdown-item" href="' + url_for('index', i=item.interview, new_session='1') + '">' + item.get_title(docassemble.base.functions.get_language()) + '</a>'
                    if app.config['SHOW_MY_INTERVIEWS'] or current_user.has_role('admin'):
                        navbar += '<a class="dropdown-item" href="' + url_for('interview_list') + '">' + word('My Interviews') + '</a>'
                    if current_user.has_role('admin', 'developer'):
                        navbar += '<a class="dropdown-item" href="' + url_for('user_profile_page') + '">' + word('Profile') + '</a>'
                    else:
                        if app.config['SHOW_PROFILE'] or current_user.has_role('admin'):
                            navbar += '<a class="dropdown-item" href="' + url_for('user_profile_page') + '">' + word('Profile') + '</a>'
                        else:
                            navbar += '<a class="dropdown-item" href="' + url_for('user.change_password') + '">' + word('Change Password') + '</a>'
                    navbar += '<a class="dropdown-item" href="' + url_for('user.logout') + '">' + word('Sign Out') + '</a>'
                navbar += '</div></li>'
    else:
        if custom_menu:
            navbar += '            <li class="nav-item dropdown"><a class="nav-link dropdown-toggle" href="#" class="dropdown-toggle d-none d-md-block" data-bs-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">' + word("Menu") + '</a><div class="dropdown-menu dropdown-menu-end">' + custom_menu
            if not status.question.interview.options.get('hide standard menu', False):
                navbar += '<a class="dropdown-item" href="' + exit_href(status) + '">' + status.exit_label + '</a>'
            navbar += '</div></li>'
        else:
            navbar += '            <li class="nav-item"><a class="nav-link" href="' + exit_href(status) + '">' + status.exit_label + '</a></li>'
    navbar += """
          </ul>"""
    if daconfig.get('login link style', 'normal') == 'button' and show_login and current_user.is_anonymous and not custom_menu:
        navbar += '\n          <a class="btn btn-' + BUTTON_COLOR_NAV_LOGIN + ' btn-sm mb-0 ms-3 d-none d-md-block" href="' + login_url + '">' + word('Sign in') + '</a>'
    navbar += """
        </div>
      </div>
    </div>
"""
    return navbar

def exit_href(status):
    exit_link = status.exit_link
    if exit_link == 'logout':
        if docassemble.base.functions.this_thread.current_info['user']['is_authenticated']:
            exit_link = 'user.logout'
        else:
            exit_link = 'leave'
    if status.exit_url:
        return url_for(exit_link, next=status.exit_url)
    return url_for(exit_link)

def delete_session_for_interview(i=None):
    if i is not None:
        clear_session(i)
    for key in ('i', 'uid', 'key_logged', 'encrypted', 'chatstatus', 'observer', 'monitor', 'doing_sms'):
        if key in session:
            del session[key]

def delete_session_sessions():
    if 'sessions' in session:
        del session['sessions']

def delete_session_info():
    for key in ('i', 'uid', 'key_logged', 'tempuser', 'user_id', 'encrypted', 'chatstatus', 'observer', 'monitor', 'variablefile', 'doing_sms', 'playgroundfile', 'playgroundtemplate', 'playgroundstatic', 'playgroundsources', 'playgroundmodules', 'playgroundpackages', 'taskwait', 'phone_number', 'otp_secret', 'validated_user', 'github_next', 'next', 'sessions'):
        if key in session:
            del session[key]

def backup_session():
    backup = {}
    for key in ('i', 'uid', 'key_logged', 'tempuser', 'user_id', 'encrypted', 'chatstatus', 'observer', 'monitor', 'variablefile', 'doing_sms', 'taskwait', 'phone_number', 'otp_secret', 'validated_user', 'github_next', 'next', 'sessions'):
        if key in session:
            backup[key] = session[key]
    return backup

def restore_session(backup):
    for key in ('i', 'uid', 'key_logged', 'tempuser', 'user_id', 'encrypted', 'google_id', 'google_email', 'chatstatus', 'observer', 'monitor', 'variablefile', 'doing_sms', 'taskwait', 'phone_number', 'otp_secret', 'validated_user', 'github_next', 'next', 'sessions'):
        if key in backup:
            session[key] = backup[key]

def get_existing_session(yaml_filename, secret):
    keys = [result.key for result in db.session.execute(select(UserDictKeys.filename, UserDictKeys.key).where(and_(UserDictKeys.user_id == current_user.id, UserDictKeys.filename == yaml_filename)).order_by(UserDictKeys.indexno))]
    for key in keys:
        try:
            steps, user_dict, is_encrypted = fetch_user_dict(key, yaml_filename, secret=secret)
        except:
            logmessage("get_existing_session: unable to decrypt existing interview session " + key)
            continue
        update_session(yaml_filename, uid=key, key_logged=True, encrypted=is_encrypted)
        return key, is_encrypted
    return None, True

def reset_session(yaml_filename, secret):
    user_dict = fresh_dictionary()
    user_code = get_unique_name(yaml_filename, secret)
    if STATS:
        r.incr('da:stats:sessions')
    update_session(yaml_filename, uid=user_code)
    return (user_code, user_dict)

def _endpoint_url(endpoint, **kwargs):
    url = url_for('index')
    if endpoint:
        url = url_for(endpoint, **kwargs)
    return url

def user_can_edit_package(pkgname=None, giturl=None):
    if current_user.has_role('admin'):
        return True
    if not PACKAGE_PROTECTION:
        if pkgname in ('docassemble.base', 'docassemble.demo', 'docassemble.webapp'):
            return False
        return True
    if pkgname is not None:
        pkgname = pkgname.strip()
        if pkgname == '' or re.search(r'\s', pkgname):
            return False
        results = db.session.execute(select(Package.id, PackageAuth.user_id, PackageAuth.authtype).outerjoin(PackageAuth, Package.id == PackageAuth.package_id).where(and_(Package.name == pkgname, Package.active == True))).all()
        the_count = 0
        for result in results:
            the_count += 1
        if the_count == 0:
            return True
        for d in results:
            if d.user_id == current_user.id:
                return True
    if giturl is not None:
        giturl = giturl.strip()
        if giturl == '' or re.search(r'\s', giturl):
            return False
        results = db.session.execute(select(Package.id, PackageAuth.user_id, PackageAuth.authtype).outerjoin(PackageAuth, Package.id == PackageAuth.package_id).where(and_(or_(Package.giturl == giturl + '/', Package.giturl == giturl), Package.active == True))).all()
        the_count = 0
        for result in results:
            the_count += 1
        if the_count == 0:
            return True
        for d in results:
            if d.user_id == current_user.id:
                return True
    return False

def uninstall_package(packagename):
    #logmessage("server uninstall_package: " + packagename)
    existing_package = db.session.execute(select(Package).filter_by(name=packagename, active=True).order_by(Package.id.desc())).first()
    if existing_package is None:
        flash(word("Package did not exist"), 'error')
        return
    db.session.execute(update(Package).where(Package.name == packagename, Package.active == True).values(active=False))
    db.session.commit()

def summarize_results(results, logmessages, html=True):
    if html:
        output = '<br>'.join([x + ':&nbsp;' + results[x] for x in sorted(results.keys())])
        if len(logmessages) > 0:
            if len(output) > 0:
                output += '<br><br><strong>'+ word("pip log") + ':</strong><br>'
            else:
                output = ''
            output += re.sub(r'\n', r'<br>', logmessages)
        return Markup(output)
    output = '\n'.join([x + ': ' + results[x] for x in sorted(results.keys())])
    if len(logmessages) > 0:
        if len(output) > 0:
            output += "\n" + word("pip log") + ':\n'
        else:
            output = ''
        output += logmessages
    return output

def install_zip_package(packagename, file_number):
    #logmessage("install_zip_package: " + packagename + " " + str(file_number))
    existing_package = db.session.execute(select(Package).filter_by(name=packagename).order_by(Package.id.desc()).with_for_update()).scalar()
    if existing_package is None:
        package_auth = PackageAuth(user_id=current_user.id)
        package_entry = Package(name=packagename, package_auth=package_auth, upload=file_number, active=True, type='zip', version=1)
        db.session.add(package_auth)
        db.session.add(package_entry)
    else:
        if existing_package.type == 'zip' and existing_package.upload is not None and existing_package.upload != file_number:
            SavedFile(existing_package.upload).delete()
        existing_package.package_auth.user_id = current_user.id
        existing_package.package_auth.authtype = 'owner'
        existing_package.upload = file_number
        existing_package.active = True
        existing_package.limitation = None
        existing_package.giturl = None
        existing_package.gitbranch = None
        existing_package.type = 'zip'
        existing_package.version += 1
    db.session.commit()

def install_git_package(packagename, giturl, branch):
    #logmessage("install_git_package: " + packagename + " " + str(giturl))
    giturl = str(giturl).rstrip('/')
    if branch is None or str(branch).lower().strip() in ('none', ''):
        branch = GITHUB_BRANCH
    if db.session.execute(select(Package).filter_by(name=packagename)).first() is None and db.session.execute(select(Package).where(or_(Package.giturl == giturl, Package.giturl == giturl + '/')).with_for_update()).scalar() is None:
        package_auth = PackageAuth(user_id=current_user.id)
        package_entry = Package(name=packagename, giturl=giturl, package_auth=package_auth, version=1, active=True, type='git', upload=None, limitation=None, gitbranch=branch)
        db.session.add(package_auth)
        db.session.add(package_entry)
    else:
        existing_package = db.session.execute(select(Package).filter_by(name=packagename).order_by(Package.id.desc()).with_for_update()).scalar()
        if existing_package is None:
            existing_package = db.session.execute(select(Package).where(or_(Package.giturl == giturl, Package.giturl == giturl + '/')).order_by(Package.id.desc()).with_for_update()).scalar()
        if existing_package is not None:
            if existing_package.type == 'zip' and existing_package.upload is not None:
                SavedFile(existing_package.upload).delete()
            existing_package.package_auth.user_id = current_user.id
            existing_package.package_auth.authtype = 'owner'
            existing_package.name = packagename
            existing_package.giturl = giturl
            existing_package.upload = None
            existing_package.version += 1
            existing_package.limitation = None
            existing_package.active = True
            if branch:
                existing_package.gitbranch = branch
            existing_package.type = 'git'
        else:
            logmessage("install_git_package: package " + str(giturl) + " appeared to exist but could not be found")
    db.session.commit()

def install_pip_package(packagename, limitation):
    #logmessage("install_pip_package: " + packagename + " " + str(limitation))
    existing_package = db.session.execute(select(Package).filter_by(name=packagename).order_by(Package.id.desc()).with_for_update()).scalar()
    if existing_package is None:
        package_auth = PackageAuth(user_id=current_user.id)
        package_entry = Package(name=packagename, package_auth=package_auth, limitation=limitation, version=1, active=True, type='pip')
        db.session.add(package_auth)
        db.session.add(package_entry)
    else:
        if existing_package.type == 'zip' and existing_package.upload is not None:
            SavedFile(existing_package.upload).delete()
        existing_package.package_auth.user_id = current_user.id
        existing_package.package_auth.authtype = 'owner'
        existing_package.version += 1
        existing_package.type = 'pip'
        existing_package.limitation = limitation
        existing_package.giturl = None
        existing_package.gitbranch = None
        existing_package.upload = None
        existing_package.active = True
    db.session.commit()

def get_package_info(exclude_core=False):
    is_admin = bool(current_user.has_role('admin'))
    package_list = []
    package_auth = {}
    seen = {}
    for auth in db.session.execute(select(PackageAuth)).scalars():
        if auth.package_id not in package_auth:
            package_auth[auth.package_id] = {}
        package_auth[auth.package_id][auth.user_id] = auth.authtype
    for package in db.session.execute(select(Package).filter_by(active=True).order_by(Package.name, Package.id.desc())).scalars():
        #if exclude_core and package.name in ('docassemble', 'docassemble.base', 'docassemble.webapp'):
        #    continue
        if package.name in seen:
            continue
        seen[package.name] = 1
        if package.type is not None:
            can_update = not bool(package.type == 'zip')
            can_uninstall = bool(is_admin or (package.id in package_auth and current_user.id in package_auth[package.id]))
            if package.core or package.name in ('docassemble', 'docassemble.base', 'docassemble.webapp'):
                can_uninstall = False
                can_update = is_admin
            if exclude_core and package.name in ('docassemble', 'docassemble.base', 'docassemble.webapp'):
                can_uninstall = False
                can_update = False
            package_list.append(Object(package=package, can_update=can_update, can_uninstall=can_uninstall))
    return package_list, package_auth

def name_of_user(user, include_email=False):
    output = ''
    if user.first_name:
        output += user.first_name
        if user.last_name:
            output += ' '
    if user.last_name:
        output += user.last_name
    if include_email and user.email:
        if output:
            output += ', '
        output += user.email
    return output

def flash_as_html(message, message_type="info", is_ajax=True):
    if message_type == 'error':
        message_type = 'danger'
    output = "\n        " + (NOTIFICATION_MESSAGE % (message_type, str(message))) + "\n"
    if not is_ajax:
        flash(message, message_type)
    return output

def make_example_html(examples, first_id, example_html, data_dict):
    example_html.append('          <ul class="nav flex-column nav-pills da-example-list da-example-hidden">\n')
    for example in examples:
        if 'list' in example:
            example_html.append('          <li class="nav-item"><a tabindex="0" class="nav-link da-example-heading">' + example['title'] + '</a>')
            make_example_html(example['list'], first_id, example_html, data_dict)
            example_html.append('          </li>')
            continue
        if len(first_id) == 0:
            first_id.append(example['id'])
        example_html.append('            <li class="nav-item"><a tabindex="0" class="nav-link da-example-link" data-example="' + example['id'] + '">' + example['title'] + '</a></li>')
        data_dict[example['id']] = example
    example_html.append('          </ul>')

def public_method(method, the_class):
    if isinstance(method, the_method_type) and method.__name__ != 'init' and not method.__name__.startswith('_') and method.__name__ in the_class.__dict__:
        return True
    return False

def noquotetrunc(string):
    string = noquote(string)
    if string is not None:
        try:
            str('') + string
        except:
            string = ''
        if len(string) > 163:
            string = string[:160] + '...'
    return string

def noquote(string):
    if string is None:
        return string
    string = amp_match.sub('&amp;', string)
    string = noquote_match.sub('&quot;', string)
    string = lt_match.sub('&lt;', string)
    string = gt_match.sub('&gt;', string)
    return string

def infobutton(title):
    docstring = ''
    if 'doc' in title_documentation[title]:
        docstring += noquote(title_documentation[title]['doc'])
    if 'url' in title_documentation[title]:
        docstring += "<br><a target='_blank' href='" + title_documentation[title]['url'] + "'>" + word("View documentation") + "</a>"
    return '&nbsp;<a tabindex="0" role="button" class="daquestionsign" data-bs-container="body" data-bs-toggle="popover" data-bs-placement="auto" data-bs-content="' + docstring + '" title="' + noquote(title_documentation[title].get('title', title)) + '"><i class="fas fa-question-circle"></i></a>'
    #title=' + json.dumps(word("Help"))
    #data-bs-selector="true"

def search_button(var, field_origins, name_origins, interview_source, all_sources):
    in_this_file = False
    usage = {}
    if var in field_origins:
        for x in sorted(field_origins[var]):
            if x is interview_source:
                in_this_file = True
            else:
                if x.path not in usage:
                    usage[x.path] = set()
                usage[x.path].add('defined')
                all_sources.add(x)
    if var in name_origins:
        for x in sorted(name_origins[var]):
            if x is interview_source:
                in_this_file = True
            else:
                if x.path not in usage:
                    usage[x.path] = set()
                usage[x.path].add('used')
                all_sources.add(x)
    usage_type = [set(), set(), set()]
    for path, the_set in usage.items():
        if 'defined' in the_set and 'used' in the_set:
            usage_type[2].add(path)
        elif 'used' in the_set:
            usage_type[1].add(path)
        elif 'defined' in the_set:
            usage_type[0].add(path)
        else:
            continue
    messages = []
    if len(usage_type[2]) > 0:
        messages.append(word("Defined and used in " + docassemble.base.functions.comma_and_list(sorted(usage_type[2]))))
    elif len(usage_type[0]) > 0:
        messages.append(word("Defined in") + ' ' + docassemble.base.functions.comma_and_list(sorted(usage_type[0])))
    elif len(usage_type[2]) > 0:
        messages.append(word("Used in") + ' ' + docassemble.base.functions.comma_and_list(sorted(usage_type[0])))
    if len(messages) > 0:
        title = 'title="' + '; '.join(messages) + '" '
    else:
        title = ''
    if in_this_file:
        classname = 'dasearchthis'
    else:
        classname = 'dasearchother'
    return '<a tabindex="0" class="dasearchicon ' + classname + '" ' + title + 'data-name="' + noquote(var) + '"><i class="fas fa-search"></i></a>'

search_key = """
                  <tr><td><h4>""" + word("Note") + """</h4></td></tr>
                  <tr><td><a tabindex="0" class="dasearchicon dasearchthis"><i class="fas fa-search"></i></a> """ + word("means the name is located in this file") + """</td></tr>
                  <tr><td><a tabindex="0" class="dasearchicon dasearchother"><i class="fas fa-search"></i></a> """ + word("means the name may be located in a file included by reference, such as:") + """</td></tr>"""

def find_needed_names(interview, needed_names, the_name=None, the_question=None):
    if the_name is not None:
        needed_names.add(the_name)
        if the_name in interview.questions:
            for lang in interview.questions[the_name]:
                for question in interview.questions[the_name][lang]:
                    find_needed_names(interview, needed_names, the_question=question)
    elif the_question is not None:
        for the_set in (the_question.mako_names, the_question.names_used):
            for name in the_set:
                if name in needed_names:
                    continue
                find_needed_names(interview, needed_names, the_name=name)
    else:
        for question in interview.questions_list:
            #if not (question.is_mandatory or question.is_initial):
            #    continue
            find_needed_names(interview, needed_names, the_question=question)

def get_ml_info(varname, default_package, default_file):
    parts = varname.split(':')
    if len(parts) == 3 and parts[0].startswith('docassemble.') and re.match(r'data/sources/.*\.json', parts[1]):
        the_package = parts[0]
        the_file = parts[1]
        the_varname = parts[2]
    elif len(parts) == 2 and parts[0] == 'global':
        the_package = '_global'
        the_file = '_global'
        the_varname = parts[1]
    elif len(parts) == 2 and (re.match(r'data/sources/.*\.json', parts[0]) or re.match(r'[^/]+\.json', parts[0])):
        the_package = default_package
        the_file = re.sub(r'^data/sources/', '', parts[0])
        the_varname = parts[1]
    elif len(parts) != 1:
        the_package = '_global'
        the_file = '_global'
        the_varname = varname
    else:
        the_package = default_package
        the_file = default_file
        the_varname = varname
    return (the_package, the_file, the_varname)

pg_code_cache = {}

def source_code_url(the_name, datatype=None):
    if datatype == 'module':
        try:
            if (not hasattr(the_name, '__path__')) or (not the_name.__path__):
                #logmessage("Nothing for module " + the_name)
                return None
            source_file = re.sub(r'\.pyc$', r'.py', the_name.__path__[0])
            line_number = 1
        except:
            return None
    elif datatype == 'class':
        try:
            source_file = inspect.getsourcefile(the_name)
            line_number = inspect.findsource(the_name)[1]
        except:
            #logmessage("Nothing for class " + the_name)
            return None
    elif hasattr(the_name, '__code__'):
        source_file = the_name.__code__.co_filename
        line_number = the_name.__code__.co_firstlineno
    else:
        #logmessage("Nothing for " + the_name)
        return None
    source_file = re.sub(r'.*/site-packages/', '', source_file)
    m = re.search(r'^docassemble/(base|webapp|demo)/', source_file)
    if m:
        output = 'https://github.com/jhpyle/docassemble/blob/master/docassemble_' + m.group(1) + '/' + source_file
        if line_number == 1:
            return output
        return output + '#L' + str(line_number)
    #logmessage("no match for " + str(source_file))
    return None

def get_vars_in_use(interview, interview_status, debug_mode=False, return_json=False, show_messages=True, show_jinja_help=False, current_project='default', use_playground=True):
    user_dict = fresh_dictionary()
    #if 'uid' not in session:
    #    session['uid'] = random_alphanumeric(32)
    if debug_mode:
        has_error = True
        error_message = "Not checking variables because in debug mode."
        error_type = Exception
    else:
        if not interview.success:
            has_error = True
            error_type = DAErrorCompileError
        else:
            old_language = docassemble.base.functions.get_language()
            try:
                interview.assemble(user_dict, interview_status)
                has_error = False
            except Exception as errmess:
                has_error = True
                error_message = str(errmess)
                error_type = type(errmess)
                logmessage("get_vars_in_use: failed assembly with error type " + str(error_type) + " and message: " + error_message)
            docassemble.base.functions.set_language(old_language)
    fields_used = set()
    names_used = set()
    field_origins = {}
    name_origins = {}
    all_sources = set()
    names_used.update(interview.names_used)
    for question in interview.questions_list:
        for the_set in (question.mako_names, question.names_used, question.fields_used):
            names_used.update(the_set)
            for key in the_set:
                if key not in name_origins:
                    name_origins[key] = set()
                name_origins[key].add(question.from_source)
        fields_used.update(question.fields_used)
        for key in question.fields_used:
            if key not in field_origins:
                field_origins[key] = set()
            field_origins[key].add(question.from_source)
            # if key == 'advocate':
            #     try:
            #         logmessage("Found advocate in " + question.content.original_text)
            #     except:
            #         logmessage("Found advocate")
    for val in interview.questions:
        names_used.add(val)
        if val not in name_origins:
            name_origins[val] = set()
        for lang in interview.questions[val]:
            for q in interview.questions[val][lang]:
                name_origins[val].add(q.from_source)
        fields_used.add(val)
        if val not in field_origins:
            field_origins[val] = set()
        for lang in interview.questions[val]:
            for q in interview.questions[val][lang]:
                field_origins[val].add(q.from_source)
    needed_names = set()
    find_needed_names(interview, needed_names)
    functions = set()
    modules = set()
    classes = set()
    name_info = copy.deepcopy(base_name_info)
    if use_playground:
        area = SavedFile(current_user.id, fix=True, section='playgroundtemplate')
        the_directory = directory_for(area, current_project)
        templates = sorted([f for f in os.listdir(the_directory) if os.path.isfile(os.path.join(the_directory, f)) and re.search(r'^[A-Za-z0-9]', f)])
        area = SavedFile(current_user.id, fix=True, section='playgroundstatic')
        the_directory = directory_for(area, current_project)
        static = sorted([f for f in os.listdir(the_directory) if os.path.isfile(os.path.join(the_directory, f)) and re.search(r'^[A-Za-z0-9]', f)])
        area = SavedFile(current_user.id, fix=True, section='playgroundsources')
        the_directory = directory_for(area, current_project)
        sources = sorted([f for f in os.listdir(the_directory) if os.path.isfile(os.path.join(the_directory, f)) and re.search(r'^[A-Za-z0-9]', f)])
        area = SavedFile(current_user.id, fix=True, section='playgroundmodules')
        the_directory = directory_for(area, current_project)
        avail_modules = sorted([re.sub(r'.py$', '', f) for f in os.listdir(the_directory) if os.path.isfile(os.path.join(the_directory, f)) and re.search(r'^[A-Za-z0-9]', f)])
    else:
        templates = []
        static = []
        sources = []
        avail_modules = []
    for val in user_dict:
        if isinstance(user_dict[val], types.FunctionType):
            if val not in pg_code_cache:
                try:
                    pg_code_cache[val] = {'doc': noquotetrunc(inspect.getdoc(user_dict[val])), 'name': str(val), 'insert': str(val) + '()', 'tag': str(val) + str(inspect.signature(user_dict[val])), 'git': source_code_url(user_dict[val])}
                except:
                    pg_code_cache[val] = {'doc': '', 'name': str(val), 'insert': str(val) + '()', 'tag': str(val) + '()', 'git': source_code_url(user_dict[val])}
            name_info[val] = copy.copy(pg_code_cache[val])
            if 'tag' in name_info[val]:
                functions.add(val)
        elif isinstance(user_dict[val], types.ModuleType):
            if val not in pg_code_cache:
                try:
                    pg_code_cache[val] = {'doc': noquotetrunc(inspect.getdoc(user_dict[val])), 'name': str(val), 'insert': str(val), 'git': source_code_url(user_dict[val], datatype='module')}
                except:
                    pg_code_cache[val] = {'doc': '', 'name': str(val), 'insert': str(val), 'git': source_code_url(user_dict[val], datatype='module')}
                name_info[val] = copy.copy(pg_code_cache[val])
                if 'git' in name_info[val]:
                    modules.add(val)
        elif isinstance(user_dict[val], TypeType):
            if val not in pg_code_cache:
                bases = []
                for x in list(user_dict[val].__bases__):
                    if x.__name__ != 'DAObject':
                        bases.append(x.__name__)
                try:
                    methods = inspect.getmembers(user_dict[val], predicate=lambda x: public_method(x, user_dict[val]))
                except:
                    methods = []
                method_list = []
                for name, value in methods:
                    try:
                        method_list.append({'insert': '.' + str(name) + '()', 'name': str(name), 'doc': noquotetrunc(inspect.getdoc(value)), 'tag': '.' + str(name) + str(inspect.signature(value)), 'git': source_code_url(value)})
                    except:
                        method_list.append({'insert': '.' + str(name) + '()', 'name': str(name), 'doc': '', 'tag': '.' + str(name) + '()', 'git': source_code_url(value)})
                try:
                    pg_code_cache[val] = {'doc': noquotetrunc(inspect.getdoc(user_dict[val])), 'name': str(val), 'insert': str(val), 'bases': bases, 'methods': method_list, 'git': source_code_url(user_dict[val], datatype='class')}
                except:
                    pg_code_cache[val] = {'doc': '', 'name': str(val), 'insert': str(val), 'bases': bases, 'methods': method_list, 'git': source_code_url(user_dict[val], datatype='class')}
            name_info[val] = copy.copy(pg_code_cache[val])
            if 'methods' in name_info[val]:
                classes.add(val)
    for val in docassemble.base.functions.pickleable_objects(user_dict):
        names_used.add(val)
        if val not in name_info:
            name_info[val] = {}
        name_info[val]['type'] = user_dict[val].__class__.__name__
        name_info[val]['iterable'] = bool(hasattr(user_dict[val], '__iter__') and not isinstance(user_dict[val], str))
    for var in base_name_info:
        if base_name_info[var]['show']:
            names_used.add(var)
    names_used = set(i for i in names_used if not extraneous_var.search(i))
    for var in ('_internal', '__object_type', '_DAOBJECTDEFAULTDA'):
        names_used.discard(var)
    for var in interview.mlfields:
        names_used.discard(var + '.text')
    if len(interview.mlfields) > 0:
        classes.add('DAModel')
        method_list = [{'insert': '.predict()', 'name': 'predict', 'doc': "Generates a prediction based on the 'text' attribute and sets the attributes 'entry_id,' 'predictions,' 'prediction,' and 'probability.'  Called automatically.", 'tag': '.predict(self)'}]
        name_info['DAModel'] = {'doc': 'Applies natural language processing to user input and returns a prediction.', 'name': 'DAModel', 'insert': 'DAModel', 'bases': [], 'methods': method_list}
    view_doc_text = word("View documentation")
    word_documentation = word("Documentation")
    attr_documentation = word("Show attributes")
    ml_parts = interview.get_ml_store().split(':')
    if len(ml_parts) == 2:
        ml_parts[1] = re.sub(r'^data/sources/ml-|\.json$', '', ml_parts[1])
    else:
        ml_parts = ['_global', '_global']
    for var in documentation_dict:
        if var not in name_info:
            name_info[var] = {}
        if 'doc' in name_info[var] and name_info[var]['doc'] is not None:
            name_info[var]['doc'] += '<br>'
        else:
            name_info[var]['doc'] = ''
        name_info[var]['doc'] += "<a target='_blank' href='" + documentation_dict[var] + "'>" + view_doc_text + "</a>"
    for var in name_info:
        if 'methods' in name_info[var]:
            for method in name_info[var]['methods']:
                if var + '.' + method['name'] in documentation_dict:
                    if method['doc'] is None:
                        method['doc'] = ''
                    else:
                        method['doc'] += '<br>'
                    if view_doc_text not in method['doc']:
                        method['doc'] += "<a target='_blank' href='" + documentation_dict[var + '.' + method['name']] + "'>" + view_doc_text + "</a>"
    content = ''
    if has_error and show_messages:
        error_style = 'danger'
        if error_type is DAErrorNoEndpoint:
            error_style = 'warning'
            message_to_use = title_documentation['incomplete']['doc']
        elif error_type is DAErrorCompileError:
            message_to_use = title_documentation['compilefail']['doc']
        elif error_type is DAErrorMissingVariable:
            message_to_use = error_message
        else:
            message_to_use = title_documentation['generic error']['doc']
        content += '\n                  <tr><td class="playground-warning-box"><div class="alert alert-' + error_style + '">' + message_to_use + '</div></td></tr>'
    vocab_dict = {}
    vocab_set = (names_used | functions | classes | modules | fields_used | set(key for key in base_name_info if not re.search(r'\.', key)) | set(key for key in name_info if not re.search(r'\.', key)) | set(templates) | set(static) | set(sources) | set(avail_modules) | set(interview.images.keys()))
    vocab_set = set(i for i in vocab_set if not extraneous_var.search(i))
    names_used = names_used.difference( functions | classes | modules | set(avail_modules) )
    undefined_names = names_used.difference(fields_used | set(base_name_info.keys()) | set(x for x in names_used if '.' in x))
    implicitly_defined = set()
    for var in fields_used:
        the_var = var
        while '.' in the_var:
            the_var = re.sub(r'(.*)\..*$', r'\1', the_var)
            implicitly_defined.add(the_var)
    for var in ('_internal', '__object_type', '_DAOBJECTDEFAULTDA'):
        undefined_names.discard(var)
        vocab_set.discard(var)
    for var in [x for x in undefined_names if x.endswith(']')]:
        undefined_names.discard(var)
    names_used = names_used.difference( undefined_names )
    if return_json:
        if len(names_used) > 0:
            has_parent = {}
            has_children = set()
            for var in names_used:
                parent = re.sub(r'[\.\[].*', '', var)
                if parent != var:
                    has_parent[var] = parent
                    has_children.add(parent)
            var_list = []
            for var in sorted(names_used):
                var_trans = re.sub(r'\[[0-9]+\]', '[i]', var)
                #var_trans = re.sub(r'\[i\](.*)\[i\](.*)\[i\](.*)\[i\](.*)\[i\](.*)\[i\]', r'[i]\1[j]\2[k]\3[l]\4[m]\5[n]', var_trans)
                #var_trans = re.sub(r'\[i\](.*)\[i\](.*)\[i\](.*)\[i\](.*)\[i\]', r'[i]\1[j]\2[k]\3[l]\4[m]', var_trans)
                #var_trans = re.sub(r'\[i\](.*)\[i\](.*)\[i\](.*)\[i\]', r'[i]\1[j]\2[k]\3[l]', var_trans)
                var_trans = re.sub(r'\[i\](.*)\[i\](.*)\[i\]', r'[i]\1[j]\2[k]', var_trans)
                var_trans = re.sub(r'\[i\](.*)\[i\]', r'[i]\1[j]', var_trans)
                info = dict(var=var, to_insert=var)
                if var_trans != var:
                    info['var_base'] = var_trans
                info['hide'] = bool(var in has_parent)
                if var in base_name_info:
                    if not base_name_info[var]['show']:
                        continue
                if var in documentation_dict or var in base_name_info:
                    info['var_type'] = 'builtin'
                elif var not in fields_used and var not in implicitly_defined and var_trans not in fields_used and var_trans not in implicitly_defined:
                    info['var_type'] = 'not_used'
                elif var not in needed_names:
                    info['var_type'] = 'possibly_not_used'
                else:
                    info['var_type'] = 'default'
                if var in name_info and 'type' in name_info[var] and name_info[var]['type']:
                    info['class_name'] = name_info[var]['type']
                elif var in interview.mlfields:
                    info['class_name'] = 'DAModel'
                if var in name_info and 'iterable' in name_info[var]:
                    info['iterable'] = name_info[var]['iterable']
                if var in name_info and 'doc' in name_info[var] and name_info[var]['doc']:
                    info['doc_content'] = name_info[var]['doc']
                    info['doc_title'] = word_documentation
                if var in interview.mlfields:
                    if 'ml_group' in interview.mlfields[var] and not interview.mlfields[var]['ml_group'].uses_mako:
                        (ml_package, ml_file, ml_group_id) = get_ml_info(interview.mlfields[var]['ml_group'].original_text, ml_parts[0], ml_parts[1])
                        info['train_link'] = url_for('train', package=ml_package, file=ml_file, group_id=ml_group_id)
                    else:
                        info['train_link'] = url_for('train', package=ml_parts[0], file=ml_parts[1], group_id=var)
                var_list.append(info)
        functions_list = []
        if len(functions) > 0:
            for var in sorted(functions):
                info = dict(var=var, to_insert=name_info[var]['insert'], name=name_info[var]['tag'])
                if 'doc' in name_info[var] and name_info[var]['doc']:
                    info['doc_content'] = name_info[var]['doc']
                    info['doc_title'] = word_documentation
                functions_list.append(info)
        classes_list = []
        if len(classes) > 0:
            for var in sorted(classes):
                info = dict(var=var, to_insert=name_info[var]['insert'], name=name_info[var]['name'])
                if name_info[var]['bases']:
                    info['bases'] = name_info[var]['bases']
                if 'doc' in name_info[var] and name_info[var]['doc']:
                    info['doc_content'] = name_info[var]['doc']
                    info['doc_title'] = word_documentation
                if 'methods' in name_info[var] and len(name_info[var]['methods']):
                    info['methods'] = []
                    for method_item in name_info[var]['methods']:
                        method_info = dict(name=method_item['name'], to_insert=method_item['insert'], tag=method_item['tag'])
                        if 'git' in method_item:
                            method_info['git'] = method_item['git']
                        if method_item['doc']:
                            method_info['doc_content'] = method_item['doc']
                            method_info['doc_title'] = word_documentation
                        info['methods'].append(method_info)
                classes_list.append(info)
        modules_list = []
        if len(modules) > 0:
            for var in sorted(modules):
                info = dict(var=var, to_insert=name_info[var]['insert'])
                if name_info[var]['doc']:
                    info['doc_content'] = name_info[var]['doc']
                    info['doc_title'] = word_documentation
                modules_list.append(info)
        if use_playground:
            modules_available_list = []
            if len(avail_modules) > 0:
                for var in sorted(avail_modules):
                    info = dict(var=var, to_insert="." + var)
                    modules_available_list.append(info)
            templates_list = []
            if len(templates) > 0:
                for var in sorted(templates):
                    info = dict(var=var, to_insert=var)
                    templates_list.append(info)
            sources_list = []
            if len(sources) > 0:
                for var in sorted(sources):
                    info = dict(var=var, to_insert=var)
                    sources_list.append(info)
            static_list = []
            if len(static) > 0:
                for var in sorted(static):
                    info = dict(var=var, to_insert=var)
                    static_list.append(info)
        images_list = []
        if len(interview.images) > 0:
            for var in sorted(interview.images):
                info = dict(var=var, to_insert=var)
                the_ref = get_url_from_file_reference(interview.images[var].get_reference())
                if the_ref:
                    info['url'] = the_ref
                images_list.append(info)
        if use_playground:
            return dict(undefined_names=list(sorted(undefined_names)), var_list=var_list, functions_list=functions_list, classes_list=classes_list, modules_list=modules_list, modules_available_list=modules_available_list, templates_list=templates_list, sources_list=sources_list, images_list=images_list, static_list=static_list), sorted(vocab_set)
        return dict(undefined_names=list(sorted(undefined_names)), var_list=var_list, functions_list=functions_list, classes_list=classes_list, modules_list=modules_list, images_list=images_list), sorted(vocab_set)
    if len(undefined_names) > 0:
        content += '\n                  <tr><td><h4>' + word('Undefined names') + infobutton('undefined') + '</h4></td></tr>'
        for var in sorted(undefined_names):
            content += '\n                  <tr><td>' + search_button(var, field_origins, name_origins, interview.source, all_sources) + '<a role="button" tabindex="0" data-name="' + noquote(var) + '" data-insert="' + noquote(var) + '" class="btn btn-danger btn-sm playground-variable">' + var + '</a></td></tr>'
            vocab_dict[var] = var
    if len(names_used) > 0:
        content += '\n                  <tr><td><h4>' + word('Variables') + infobutton('variables') + '</h4></td></tr>'
        has_parent = {}
        has_children = set()
        for var in names_used:
            parent = re.sub(r'[\.\[].*', '', var)
            if parent != var:
                has_parent[var] = parent
                has_children.add(parent)
        for var in sorted(names_used):
            var_trans = re.sub(r'\[[0-9]\]', '[i]', var)
            var_trans = re.sub(r'\[i\](.*)\[i\](.*)\[i\]', r'[i]\1[j]\2[k]', var_trans)
            var_trans = re.sub(r'\[i\](.*)\[i\]', r'[i]\1[j]', var_trans)
            if var in has_parent:
                hide_it = ' style="display: none" data-parent="' + noquote(has_parent[var]) + '"'
            else:
                hide_it = ''
            if var in base_name_info:
                if not base_name_info[var]['show']:
                    continue
            if var in documentation_dict or var in base_name_info:
                class_type = 'btn-info'
                title = 'title=' + json.dumps(word("Special variable")) + ' '
            elif var not in fields_used and var not in implicitly_defined and var_trans not in fields_used and var_trans not in implicitly_defined:
                class_type = 'btn-secondary'
                title = 'title=' + json.dumps(word("Possibly not defined")) + ' '
            elif var not in needed_names:
                class_type = 'btn-warning'
                title = 'title=' + json.dumps(word("Possibly not used")) + ' '
            else:
                class_type = 'btn-primary'
                title = ''
            content += '\n                  <tr' + hide_it + '><td>' + search_button(var, field_origins, name_origins, interview.source, all_sources) + '<a role="button" tabindex="0" data-name="' + noquote(var) + '" data-insert="' + noquote(var) + '" ' + title + 'class="btn btn-sm ' + class_type + ' playground-variable">' + var + '</a>'
            vocab_dict[var] = var
            if var in has_children:
                content += '&nbsp;<a tabindex="0" class="dashowattributes" role="button" data-name="' + noquote(var) + '" title=' + json.dumps(attr_documentation) + '><i class="fas fa-ellipsis-h"></i></a>'
            if var in name_info and 'type' in name_info[var] and name_info[var]['type']:
                content +='&nbsp;<span data-ref="' + noquote(name_info[var]['type']) + '" class="daparenthetical">(' + name_info[var]['type'] + ')</span>'
            elif var in interview.mlfields:
                content +='&nbsp;<span data-ref="DAModel" class="daparenthetical">(DAModel)</span>'
            if var in name_info and 'doc' in name_info[var] and name_info[var]['doc']:
                if 'git' in name_info[var] and name_info[var]['git']:
                    git_link = noquote("<a class='float-end' target='_blank' href='" + name_info[var]['git'] + "'><i class='fas fa-code'></i></a>")
                else:
                    git_link = ''
                content += '&nbsp;<a tabindex="0" class="dainfosign" role="button" data-bs-container="body" data-bs-toggle="popover" data-bs-placement="auto" data-bs-content="' + name_info[var]['doc'] + '"  title="' + var + git_link + '"><i class="fas fa-info-circle"></i></a>' #data-bs-selector="true" title=' + json.dumps(word_documentation) + '
            if var in interview.mlfields:
                if 'ml_group' in interview.mlfields[var] and not interview.mlfields[var]['ml_group'].uses_mako:
                    (ml_package, ml_file, ml_group_id) = get_ml_info(interview.mlfields[var]['ml_group'].original_text, ml_parts[0], ml_parts[1])
                    content += '&nbsp;<a class="datrain" target="_blank" href="' + url_for('train', package=ml_package, file=ml_file, group_id=ml_group_id) + '" title=' + json.dumps(word("Train")) + '><i class="fas fa-graduation-cap"></i></a>'
                else:
                    content += '&nbsp;<a class="datrain" target="_blank" href="' + url_for('train', package=ml_parts[0], file=ml_parts[1], group_id=var) + '" title=' + json.dumps(word("Train")) + '><i class="fas fa-graduation-cap"></i></a>'
            content += '</td></tr>'
        if len(all_sources) > 0 and show_messages:
            content += search_key
            content += '\n                <tr><td>'
            content += '\n                  <ul>'
            for path in sorted([x.path for x in all_sources]):
                content += '\n                    <li><a target="_blank" href="' + url_for('view_source', i=path, project=current_project)+ '">' + path + '<a></li>'
            content += '\n                  </ul>'
            content += '\n                </td></tr>'
    if len(functions) > 0:
        content += '\n                  <tr><td><h4>' + word('Functions') + infobutton('functions') + '</h4></td></tr>'
        for var in sorted(functions):
            if var in name_info:
                content += '\n                  <tr><td><a role="button" tabindex="0" data-name="' + noquote(var) + '" data-insert="' + noquote(name_info[var]['insert']) + '" class="btn btn-sm btn-warning playground-variable">' + name_info[var]['tag'] + '</a>'
            vocab_dict[var] = name_info[var]['insert']
            if var in name_info and 'doc' in name_info[var] and name_info[var]['doc']:
                if 'git' in name_info[var] and name_info[var]['git']:
                    git_link = noquote("<a class='float-end' target='_blank' href='" + name_info[var]['git'] + "'><i class='fas fa-code'></i></a>")
                else:
                    git_link = ''
                content += '&nbsp;<a tabindex="0" class="dainfosign" role="button" data-bs-container="body" data-bs-toggle="popover" data-bs-placement="auto" data-bs-content="' + name_info[var]['doc'] + '" title="' + var + git_link + '"><i class="fas fa-info-circle"></i></a>' #data-bs-selector="true" title=' + json.dumps(word_documentation) + '
            content += '</td></tr>'
    if len(classes) > 0:
        content += '\n                  <tr><td><h4>' + word('Classes') + infobutton('classes') + '</h4></td></tr>'
        for var in sorted(classes):
            content += '\n                  <tr><td><a role="button" tabindex="0" data-name="' + noquote(var) + '" data-insert="' + noquote(name_info[var]['insert']) + '" class="btn btn-sm btn-info playground-variable">' + name_info[var]['name'] + '</a>'
            vocab_dict[var] = name_info[var]['insert']
            if name_info[var]['bases']:
                content += '&nbsp;<span data-ref="' + noquote(name_info[var]['bases'][0]) + '" class="daparenthetical">(' + name_info[var]['bases'][0] + ')</span>'
            if name_info[var]['doc']:
                if 'git' in name_info[var] and name_info[var]['git']:
                    git_link = noquote("<a class='float-end' target='_blank' href='" + name_info[var]['git'] + "'><i class='fas fa-code'></i></a>")
                else:
                    git_link = ''
                content += '&nbsp;<a tabindex="0" class="dainfosign" role="button" data-bs-container="body" data-bs-toggle="popover" data-bs-placement="auto" data-bs-content="' + name_info[var]['doc'] + '" title="' + var + git_link + '"><i class="fas fa-info-circle"></i></a>' #data-bs-selector="true" title=' + json.dumps(word_documentation) + '
            if len(name_info[var]['methods']) > 0:
                content += '&nbsp;<a tabindex="0" class="dashowmethods" role="button" data-showhide="XMETHODX' + var + '" title=' + json.dumps(word('Methods')) + '><i class="fas fa-cog"></i></a>'
                content += '<div style="display: none;" id="XMETHODX' + var + '"><table><tbody>'
                for method_info in name_info[var]['methods']:
                    if 'git' in method_info and method_info['git']:
                        git_link = noquote("<a class='float-end' target='_blank' href='" + method_info['git'] + "'><i class='fas fa-code'></i></a>")
                    else:
                        git_link = ''
                    content += '<tr><td><a tabindex="0" role="button" data-name="' + noquote(method_info['name']) + '" data-insert="' + noquote(method_info['insert']) + '" class="btn btn-sm btn-warning playground-variable">' + method_info['tag'] + '</a>'
                    #vocab_dict[method_info['name']] = method_info['insert']
                    if method_info['doc']:
                        content += '&nbsp;<a tabindex="0" class="dainfosign" role="button" data-bs-container="body" data-bs-toggle="popover" data-bs-placement="auto" data-bs-content="' + method_info['doc'] + '" data-bs-title="' + noquote(method_info['name']) + git_link + '"><i class="fas fa-info-circle"></i></a>' #data-bs-selector="true" title=' + json.dumps(word_documentation) + '
                    content += '</td></tr>'
                content += '</tbody></table></div>'
            content += '</td></tr>'
    if len(modules) > 0:
        content += '\n                  <tr><td><h4>' + word('Modules defined') + infobutton('modules') + '</h4></td></tr>'
        for var in sorted(modules):
            content += '\n                  <tr><td><a tabindex="0" data-name="' + noquote(var) + '" data-insert="' + noquote(name_info[var]['insert']) + '" role="button" class="btn btn-sm btn-success playground-variable">' + name_info[var]['name'] + '</a>'
            vocab_dict[var] = name_info[var]['insert']
            if name_info[var]['doc']:
                if 'git' in name_info[var] and name_info[var]['git']:
                    git_link = noquote("<a class='float-end' target='_blank' href='" + name_info[var]['git'] + "'><i class='fas fa-code'></i></a>")
                else:
                    git_link = ''
                content += '&nbsp;<a tabindex="0" class="dainfosign" role="button" data-bs-container="body" data-bs-toggle="popover" data-bs-placement="auto" data-bs-content="' + name_info[var]['doc'] + '" data-bs-title="' + noquote(var) + git_link + '"><i class="fas fa-info-circle"></i></a>' #data-bs-selector="true" title=' + json.dumps(word_documentation) + '
            content += '</td></tr>'
    if len(avail_modules) > 0:
        content += '\n                  <tr><td><h4>' + word('Modules available in Playground') + infobutton('playground_modules') + '</h4></td></tr>'
        for var in avail_modules:
            content += '\n                  <tr><td><a role="button" tabindex="0" data-name="' + noquote(var) + '" data-insert=".' + noquote(var) + '" class="btn btn-sm btn-success playground-variable">.' + noquote(var) + '</a>'
            vocab_dict[var] = var
            content += '</td></tr>'
    if len(templates) > 0:
        content += '\n                  <tr><td><h4>' + word('Templates') + infobutton('templates') + '</h4></td></tr>'
        for var in templates:
            content += '\n                  <tr><td><a role="button" tabindex="0" data-name="' + noquote(var) + '" data-insert="' + noquote(var) + '" class="btn btn-sm btn-secondary playground-variable">' + noquote(var) + '</a>'
            vocab_dict[var] = var
            content += '</td></tr>'
    if len(static) > 0:
        content += '\n                  <tr><td><h4>' + word('Static files') + infobutton('static') + '</h4></td></tr>'
        for var in static:
            content += '\n                  <tr><td><a role="button" tabindex="0" data-name="' + noquote(var) + '" data-insert="' + noquote(var) + '" class="btn btn-sm btn-secondary playground-variable">' + noquote(var) + '</a>'
            vocab_dict[var] = var
            content += '</td></tr>'
    if len(sources) > 0:
        content += '\n                  <tr><td><h4>' + word('Source files') + infobutton('sources') + '</h4></td></tr>'
        for var in sources:
            content += '\n                  <tr><td><a role="button" tabindex="0" data-name="' + noquote(var) + '" data-insert="' + noquote(var) + '" class="btn btn-sm btn-secondary playground-variable">' + noquote(var) + '</a>'
            vocab_dict[var] = var
            content += '</td></tr>'
    if len(interview.images) > 0:
        content += '\n                  <tr><td><h4>' + word('Decorations') + infobutton('decorations') + '</h4></td></tr>'
        show_images = not bool(cloud and len(interview.images) > 10)
        for var in sorted(interview.images):
            content += '\n                  <tr><td>'
            the_ref = get_url_from_file_reference(interview.images[var].get_reference())
            if the_ref is None:
                content += '<a role="button" tabindex="0" title=' + json.dumps(word("This image file does not exist")) + ' data-name="' + noquote(var) + '" data-insert="' + noquote(var) + '" class="btn btn-sm btn-danger playground-variable">' + noquote(var) + '</a>'
            else:
                if show_images:
                    content += '<img class="daimageicon" src="' + the_ref + '">&nbsp;'
                content += '<a role="button" tabindex="0" data-name="' + noquote(var) + '" data-insert="' + noquote(var) + '" class="btn btn-sm btn-primary playground-variable">' + noquote(var) + '</a>'
            vocab_dict[var] = var
            content += '</td></tr>'
    if show_messages:
        content += "\n                  <tr><td><br><em>" + word("Type Ctrl-space to autocomplete.") + "</em></td><tr>"
    if show_jinja_help:
        content += "\n                  <tr><td><h4 class=\"mt-2\">" + word("Using Jinja2") + infobutton('jinja2') + "</h4>\n                  " + re.sub("table-striped", "table-bordered", docassemble.base.util.markdown_to_html(word("Jinja2 help template"), trim=False, do_terms=False)) + "</td><tr>"
    for item in base_name_info:
        if item not in vocab_dict and not base_name_info.get('exclude', False):
            vocab_dict[item] = base_name_info.get('insert', item)
    return content, sorted(vocab_set), vocab_dict

def make_png_for_pdf(doc, prefix, page=None):
    if prefix == 'page':
        resolution = PNG_RESOLUTION
    else:
        resolution = PNG_SCREEN_RESOLUTION
    session_id = docassemble.base.functions.get_uid()
    task = docassemble.webapp.worker.make_png_for_pdf.delay(doc, prefix, resolution, session_id, PDFTOPPM_COMMAND, page=page)
    return task.id

def fg_make_png_for_pdf(doc, prefix, page=None):
    if prefix == 'page':
        resolution = PNG_RESOLUTION
    else:
        resolution = PNG_SCREEN_RESOLUTION
    docassemble.base.util.make_png_for_pdf(doc, prefix, resolution, PDFTOPPM_COMMAND, page=page)

def fg_make_png_for_pdf_path(path, prefix, page=None):
    if prefix == 'page':
        resolution = PNG_RESOLUTION
    else:
        resolution = PNG_SCREEN_RESOLUTION
    docassemble.base.util.make_png_for_pdf_path(path, prefix, resolution, PDFTOPPM_COMMAND, page=page)

def fg_make_pdf_for_word_path(path, extension):
    success = docassemble.base.pandoc.word_to_pdf(path, extension, path + ".pdf")
    if not success:
        raise DAError("fg_make_pdf_for_word_path: unable to make PDF from " + path + " using extension " + extension + " and writing to " + path + ".pdf")

def task_ready(task_id):
    result = docassemble.webapp.worker.workerapp.AsyncResult(id=task_id)
    if result.ready():
        return True
    return False

def wait_for_task(task_id, timeout=None):
    if timeout is None:
        timeout = 3
    #logmessage("wait_for_task: starting")
    try:
        result = docassemble.webapp.worker.workerapp.AsyncResult(id=task_id)
        if result.ready():
            #logmessage("wait_for_task: was ready")
            return True
        #logmessage("wait_for_task: waiting for task to complete")
        result.get(timeout=timeout)
        #logmessage("wait_for_task: returning true")
        return True
    except docassemble.webapp.worker.TimeoutError:
        logmessage("wait_for_task: timed out")
        return False
    except Exception as the_error:
        logmessage("wait_for_task: got error: " + str(the_error))
        return False

# def make_image_files(path):
#     if PDFTOPPM_COMMAND is not None:
#         args = [PDFTOPPM_COMMAND, '-r', str(PNG_RESOLUTION), '-png', path, path + 'page']
#         result = call(args)
#         if result > 0:
#             raise DAError("Call to pdftoppm failed")
#         args = [PDFTOPPM_COMMAND, '-r', str(PNG_SCREEN_RESOLUTION), '-png', path, path + 'screen']
#         result = call(args)
#         if result > 0:
#             raise DAError("Call to pdftoppm failed")

def trigger_update(except_for=None):
    sys.stderr.write("trigger_update: except_for is " + str(except_for) + " and hostname is " + hostname + "\n")
    if USING_SUPERVISOR:
        to_delete = set()
        for host in db.session.execute(select(Supervisors)).scalars():
            if host.url and not (except_for and host.hostname == except_for):
                if host.hostname == hostname:
                    the_url = 'http://localhost:9001'
                    sys.stderr.write("trigger_update: using http://localhost:9001\n")
                else:
                    the_url = host.url
                args = [SUPERVISORCTL, '-s', the_url, 'start', 'update']
                result = subprocess.run(args, check=False).returncode
                if result == 0:
                    sys.stderr.write("trigger_update: sent update to " + str(host.hostname) + " using " + the_url + "\n")
                else:
                    sys.stderr.write("trigger_update: call to supervisorctl on " + str(host.hostname) + " was not successful\n")
                    to_delete.add(host.id)
        for id_to_delete in to_delete:
            db.session.execute(delete(Supervisors).filter_by(id=id_to_delete))
            db.session.commit()

def restart_on(host):
    logmessage("restart_on: " + str(host.hostname))
    if host.hostname == hostname:
        the_url = 'http://localhost:9001'
    else:
        the_url = host.url
    args = [SUPERVISORCTL, '-s', the_url, 'start', 'reset']
    result = subprocess.run(args, check=False).returncode
    if result == 0:
        logmessage("restart_on: sent reset to " + str(host.hostname))
    else:
        logmessage("restart_on: call to supervisorctl with reset on " + str(host.hostname) + " was not successful")
        return False
    return True

def restart_all():
    for interview_path in [x.decode() for x in r.keys('da:interviewsource:*')]:
        r.delete(interview_path)
    restart_others()
    restart_this()

def restart_this():
    logmessage("restart_this: hostname is " + str(hostname))
    if USING_SUPERVISOR:
        to_delete = set()
        for host in db.session.execute(select(Supervisors)).scalars():
            if host.url:
                logmessage("restart_this: considering " + str(host.hostname) + " against " + str(hostname))
                if host.hostname == hostname:
                    result = restart_on(host)
                    if not result:
                        to_delete.add(host.id)
        for id_to_delete in to_delete:
            db.session.execute(delete(Supervisors).filter_by(id=id_to_delete))
            db.session.commit()
    else:
        logmessage("restart_this: touching wsgi file")
        wsgi_file = WEBAPP_PATH
        if os.path.isfile(wsgi_file):
            with open(wsgi_file, 'a', encoding='utf-8'):
                os.utime(wsgi_file, None)

def restart_others():
    logmessage("restart_others: starting")
    if USING_SUPERVISOR:
        cron_key = 'da:cron_restart'
        cron_url = None
        to_delete = set()
        for host in db.session.execute(select(Supervisors)).scalars():
            if host.url and host.hostname != hostname and ':cron:' in str(host.role):
                pipe = r.pipeline()
                pipe.set(cron_key, 1)
                pipe.expire(cron_key, 10)
                pipe.execute()
                result = restart_on(host)
                if not result:
                    to_delete.add(host.id)
                while r.get(cron_key) is not None:
                    time.sleep(1)
                cron_url = host.url
        for host in db.session.execute(select(Supervisors)).scalars():
            if host.url and host.url != cron_url and host.hostname != hostname and host.id not in to_delete:
                result = restart_on(host)
                if not result:
                    to_delete.add(host.id)
        for id_to_delete in to_delete:
            db.session.execute(delete(Supervisors).filter_by(id=id_to_delete))
            db.session.commit()

def get_requester_ip(req):
    if not req:
        return '127.0.0.1'
    if HTTP_TO_HTTPS:
        if 'X-Real-Ip' in req.headers:
            return req.headers['X-Real-Ip']
        if 'X-Forwarded-For' in req.headers:
            return req.headers['X-Forwarded-For']
    return req.remote_addr

def current_info(yaml=None, req=None, action=None, location=None, interface='web', session_info=None, secret=None, device_id=None, session_uid=None):
    #logmessage("interface is " + str(interface))
    if current_user.is_authenticated and not current_user.is_anonymous:
        role_list = [role.name for role in current_user.roles]
        if len(role_list) == 0:
            role_list = ['user']
        ext = dict(email=current_user.email, roles=role_list, the_user_id=current_user.id, theid=current_user.id, firstname=current_user.first_name, lastname=current_user.last_name, nickname=current_user.nickname, country=current_user.country, subdivisionfirst=current_user.subdivisionfirst, subdivisionsecond=current_user.subdivisionsecond, subdivisionthird=current_user.subdivisionthird, organization=current_user.organization, timezone=current_user.timezone, language=current_user.language)
    else:
        ext = dict(email=None, the_user_id='t' + str(session.get('tempuser', None)), theid=session.get('tempuser', None), roles=[])
    headers = {}
    if req is None:
        url_root = daconfig.get('url root', 'http://localhost') + ROOT
        url = url_root + 'interview'
        clientip = None
        method = None
        session_uid = '0'
    else:
        url_root = url_for('rootindex', _external=True)
        url = url_root + 'interview'
        if secret is None:
            secret = req.cookies.get('secret', None)
        for key, value in req.headers.items():
            headers[key] = value
        clientip = get_requester_ip(req)
        method = req.method
        if session_uid is None:
            if 'session' in req.cookies:
                session_uid = str(req.cookies.get('session'))[5:15]
            else:
                session_uid = ''
            if session_uid == '':
                session_uid = app.session_interface.manual_save_session(app, session).decode()[5:15]
        #logmessage("unique id is " + session_uid)
    if device_id is None:
        device_id = random_string(16)
    if secret is not None:
        secret = str(secret)
    if session_info is None and yaml is not None:
        session_info = get_session(yaml)
    if session_info is not None:
        user_code = session_info['uid']
        encrypted = session_info['encrypted']
    else:
        user_code = None
        encrypted = True
    return_val = {'session': user_code, 'secret': secret, 'yaml_filename': yaml, 'interface': interface, 'url': url, 'url_root': url_root, 'encrypted': encrypted, 'user': {'is_anonymous': current_user.is_anonymous, 'is_authenticated': current_user.is_authenticated, 'session_uid': session_uid, 'device_id': device_id}, 'headers': headers, 'clientip': clientip, 'method': method}
    if action is not None:
        #logmessage("current_info: setting an action " + repr(action))
        return_val.update(action)
        # return_val['orig_action'] = action['action']
        # return_val['orig_arguments'] = action['arguments']
    if location is not None:
        ext['location'] = location
    else:
        ext['location'] = None
    return_val['user'].update(ext)
    return return_val

def html_escape(text):
    text = re.sub('&', '&amp;', text)
    text = re.sub('<', '&lt;', text)
    text = re.sub('>', '&gt;', text)
    return text

def indent_by(text, num):
    if not text:
        return ""
    return (" " * num) + re.sub(r'\n', "\n" + (" " * num), text).rstrip() + "\n"

def call_sync():
    if not USING_SUPERVISOR:
        return
    args = [SUPERVISORCTL, '-s', 'http://localhost:9001', 'start', 'sync']
    result = subprocess.run(args, check=False).returncode
    if result == 0:
        pass
        #logmessage("call_sync: sent message to " + hostname)
    else:
        logmessage("call_sync: call to supervisorctl on " + hostname + " was not successful")
        abort(404)
    in_process = 1
    counter = 10
    check_args = [SUPERVISORCTL, '-s', 'http://localhost:9001', 'status', 'sync']
    while in_process == 1 and counter > 0:
        output, err = Popen(check_args, stdout=PIPE, stderr=PIPE).communicate()
        if not re.search(r'RUNNING', output.decode()):
            in_process = 0
        else:
            time.sleep(1)
        counter -= 1

def formatted_current_time():
    if current_user.timezone:
        the_timezone = pytz.timezone(current_user.timezone)
    else:
        the_timezone = pytz.timezone(get_default_timezone())
    return datetime.datetime.utcnow().replace(tzinfo=tz.tzutc()).astimezone(the_timezone).strftime('%H:%M:%S %Z')

def formatted_current_date():
    if current_user.timezone:
        the_timezone = pytz.timezone(current_user.timezone)
    else:
        the_timezone = pytz.timezone(get_default_timezone())
    return datetime.datetime.utcnow().replace(tzinfo=tz.tzutc()).astimezone(the_timezone).strftime("%Y-%m-%d")

class Object:
    def __init__(self, **kwargs):
        for key, value in kwargs.items():
            setattr(self, key, value)
    pass

class FakeUser:
    pass

class FakeRole:
    pass

class OAuthSignIn:
    providers = {}
    providers_obtained = False

    def __init__(self, provider_name):
        self.provider_name = provider_name
        credentials = current_app.config['OAUTH_CREDENTIALS'].get(provider_name, {})
        self.consumer_id = credentials.get('id', None)
        self.consumer_secret = credentials.get('secret', None)
        self.consumer_domain = credentials.get('domain', None)

    def authorize(self):
        pass

    def callback(self):
        pass

    def get_callback_url(self):
        return url_for('oauth_callback', provider=self.provider_name,
                       _external=True)

    @classmethod
    def get_provider(cls, provider_name):
        if not cls.providers_obtained:
            for provider_class in cls.__subclasses__():
                provider = provider_class()
                cls.providers[provider.provider_name] = provider
            cls.providers_obtained = True
        return cls.providers[provider_name]

class GoogleSignIn(OAuthSignIn):
    def __init__(self):
        super().__init__('google')
        self.service = OAuth2Service(
            name='google',
            client_id=self.consumer_id,
            client_secret=self.consumer_secret,
            authorize_url=None,
            access_token_url=None,
            base_url=None
        )
    def authorize(self):
        result = urllib.parse.parse_qs(request.data.decode())
        # logmessage("GoogleSignIn, args: " + str([str(arg) + ": " + str(request.args[arg]) for arg in request.args]))
        # logmessage("GoogleSignIn, request: " + str(request.data))
        # logmessage("GoogleSignIn, result: " + repr(raw_result))
        session['google_id'] = result.get('id', [None])[0]
        session['google_email'] = result.get('email', [None])[0]
        session['google_name'] = result.get('name', [None])[0]
        response = make_response(json.dumps('Successfully connected user.'), 200)
        response.headers['Content-Type'] = 'application/json'
        # oauth_session = self.service.get_auth_session(
        #     data={'code': request.args['code'],
        #           'grant_type': 'authorization_code',
        #           'redirect_uri': self.get_callback_url()}
        # )
        return response

    def callback(self):
        #logmessage("GoogleCallback, args: " + str([str(arg) + ": " + str(request.args[arg]) for arg in request.args]))
        #logmessage("GoogleCallback, request: " + str(request.data))
        email = session.get('google_email', None)
        google_id = session.get('google_id', None)
        google_name = session.get('google_name', None)
        if 'google_id' in session:
            del session['google_id']
        if 'google_email' in session:
            del session['google_email']
        if 'google_name' in session:
            del session['google_name']
        if email is not None and google_id is not None:
            return (
                'google$' + str(google_id),
                email.split('@')[0],
                email,
                {'name': google_name}
            )
        raise Exception("Could not get Google authorization information")

class FacebookSignIn(OAuthSignIn):
    def __init__(self):
        super().__init__('facebook')
        self.service = OAuth2Service(
            name='facebook',
            client_id=self.consumer_id,
            client_secret=self.consumer_secret,
            authorize_url='https://www.facebook.com/v3.0/dialog/oauth',
            access_token_url='https://graph.facebook.com/v3.0/oauth/access_token',
            base_url='https://graph.facebook.com/v3.0'
        )
    def authorize(self):
        return redirect(self.service.get_authorize_url(
            scope='public_profile,email',
            response_type='code',
            redirect_uri=self.get_callback_url())
        )
    def callback(self):
        if 'code' not in request.args:
            return None, None, None, None
        oauth_session = self.service.get_auth_session(
            decoder=safe_json_loads,
            data={'code': request.args['code'],
                  'redirect_uri': self.get_callback_url()}
        )
        me = oauth_session.get('me', params={'fields': 'id,name,first_name,middle_name,last_name,name_format,email'}).json()
        #logmessage("Facebook: returned " + json.dumps(me))
        return (
            'facebook$' + str(me['id']),
            me.get('email').split('@')[0],
            me.get('email'),
            {'first': me.get('first_name', None),
             'middle': me.get('middle_name', None),
             'last': me.get('last_name', None),
             'name': me.get('name', None),
             'name_format': me.get('name_format', None)}
        )

class AzureSignIn(OAuthSignIn):
    def __init__(self):
        super().__init__('azure')
        self.service = OAuth2Service(
            name='azure',
            client_id=self.consumer_id,
            client_secret=self.consumer_secret,
            authorize_url='https://login.microsoftonline.com/common/oauth2/authorize',
            access_token_url='https://login.microsoftonline.com/common/oauth2/token',
            base_url='https://graph.microsoft.com/v1.0/'
        )
    def authorize(self):
        return redirect(self.service.get_authorize_url(
            response_type='code',
            client_id=self.consumer_id,
            redirect_uri=self.get_callback_url())
        )
    def callback(self):
        if 'code' not in request.args:
            return None, None, None, None
        oauth_session = self.service.get_auth_session(
            decoder=safe_json_loads,
            data={'code': request.args['code'],
                  'client_id': self.consumer_id,
                  'client_secret': self.consumer_secret,
                  'resource': 'https://graph.microsoft.com/',
                  'grant_type': 'authorization_code',
                  'redirect_uri': self.get_callback_url()}
        )
        me = oauth_session.get('me').json()
        return (
            'azure$' + str(me['id']),
            me.get('mail').split('@')[0],
            me.get('mail'),
            {'first_name': me.get('givenName', None),
             'last_name': me.get('surname', None),
             'name': me.get('displayName', me.get('userPrincipalName', None))}
        )

def safe_json_loads(data):
    return json.loads(data.decode("utf-8", "strict"))

class Auth0SignIn(OAuthSignIn):
    def __init__(self):
        super().__init__('auth0')
        self.service = OAuth2Service(
            name='auth0',
            client_id=self.consumer_id,
            client_secret=self.consumer_secret,
            authorize_url='https://' + str(self.consumer_domain) + '/authorize',
            access_token_url='https://' + str(self.consumer_domain) + '/oauth/token',
            base_url='https://' + str(self.consumer_domain)
        )
    def authorize(self):
        if 'oauth' in daconfig and 'auth0' in daconfig['oauth'] and daconfig['oauth']['auth0'].get('enable', True) and self.consumer_domain is None:
            raise Exception("To use Auth0, you need to set your domain in the configuration.")
        return redirect(self.service.get_authorize_url(
            response_type='code',
            scope='openid profile email',
            audience='https://' + str(self.consumer_domain) + '/userinfo',
            redirect_uri=self.get_callback_url())
        )
    def callback(self):
        if 'code' not in request.args:
            return None, None, None, None
        oauth_session = self.service.get_auth_session(
            decoder=safe_json_loads,
            data={'code': request.args['code'],
                  'grant_type': 'authorization_code',
                  'redirect_uri': self.get_callback_url()}
        )
        me = oauth_session.get('userinfo').json()
        #logmessage("Auth0 returned " + json.dumps(me))
        user_id = me.get('sub', me.get('user_id'))
        social_id = 'auth0$' + str(user_id)
        username = me.get('name')
        email = me.get('email')
        if user_id is None or username is None or email is None:
            raise Exception("Error: could not get necessary information from Auth0")
        return social_id, username, email, {'name': me.get('name', None)}

class KeycloakSignIn(OAuthSignIn):
    def __init__(self):
        super().__init__('keycloak')
        try:
            realm = daconfig['oauth']['keycloak']['realm']
        except:
            realm = None
        self.service = OAuth2Service(
            name='keycloak',
            client_id=self.consumer_id,
            client_secret=self.consumer_secret,
            authorize_url='https://' + str(self.consumer_domain) + '/auth/realms/' + str(realm) + '/protocol/openid-connect/auth',
            access_token_url='https://' + str(self.consumer_domain) + '/auth/realms/' + str(realm) + '/protocol/openid-connect/token',
            base_url='https://' + str(self.consumer_domain)
        )
    def authorize(self):
        if 'oauth' in daconfig and 'keycloak' in daconfig['oauth'] and daconfig['oauth']['keycloak'].get('enable', True) and self.consumer_domain is None:
            raise Exception("To use keycloak, you need to set your domain in the configuration.")
        return redirect(self.service.get_authorize_url(
            response_type='code',
            scope='openid profile email',
            redirect_uri=self.get_callback_url())
        )
    def callback(self):
        if 'code' not in request.args:
            return None, None, None, None
        oauth_session = self.service.get_auth_session(
            decoder=safe_json_loads,
            data={'code': request.args['code'],
                  'grant_type': 'authorization_code',
                  'redirect_uri': self.get_callback_url()}
        )
        me = oauth_session.get('auth/realms/' + daconfig['oauth']['keycloak']['realm'] + '/protocol/openid-connect/userinfo').json()
        #logmessage("keycloak returned " + json.dumps(me))
        user_id = me.get('sub')
        social_id = 'keycloak$' + str(user_id)
        username = me.get('preferred_username')
        email = me.get('email')
        if user_id is None or username is None or email is None:
            raise Exception("Error: could not get necessary information from keycloak")
        return social_id, username, email, {'name': me.get('name', None)}

class TwitterSignIn(OAuthSignIn):
    def __init__(self):
        super().__init__('twitter')
        self.service = OAuth1Service(
            name='twitter',
            consumer_key=self.consumer_id,
            consumer_secret=self.consumer_secret,
            request_token_url='https://api.twitter.com/oauth/request_token',
            authorize_url='https://api.twitter.com/oauth/authorize',
            access_token_url='https://api.twitter.com/oauth/access_token',
            base_url='https://api.twitter.com/1.1/'
        )
    def authorize(self):
        request_token = self.service.get_request_token(
            params={'oauth_callback': self.get_callback_url()}
        )
        session['request_token'] = request_token
        return redirect(self.service.get_authorize_url(request_token[0]))
    def callback(self):
        request_token = session.pop('request_token')
        if 'oauth_verifier' not in request.args:
            return None, None, None, None
        oauth_session = self.service.get_auth_session(
            request_token[0],
            request_token[1],
            data={'oauth_verifier': request.args['oauth_verifier']}
        )
        me = oauth_session.get('account/verify_credentials.json', params={'skip_status': 'true', 'include_email': 'true', 'include_entites': 'false'}).json()
        #logmessage("Twitter returned " + json.dumps(me))
        social_id = 'twitter$' + str(me.get('id_str'))
        username = me.get('screen_name')
        email = me.get('email')
        return social_id, username, email, {'name': me.get('name', None)}

@flaskbabel.localeselector
def get_locale():
    translations = [str(translation) for translation in flaskbabel.list_translations()]
    return request.accept_languages.best_match(translations)

def get_user_object(user_id):
    the_user = db.session.execute(select(UserModel).options(db.joinedload(UserModel.roles)).where(UserModel.id == user_id)).scalar()
    return the_user

@lm.user_loader
def load_user(the_id):
    return UserModel.query.options(db.joinedload(UserModel.roles)).get(int(the_id))

@app.route('/goto', methods=['GET'])
def run_temp():
    code = request.args.get('c', None)
    if code is None:
        abort(403)
    the_key = 'da:temporary_url:' + str(code)
    data = r.get(the_key)
    if data is None:
        raise DAError(word("The link has expired."), code=403)
    try:
        data = json.loads(data.decode())
        if data.get('once', False):
            r.delete(the_key)
        url = data.get('url')
    except:
        r.delete(the_key)
        url = data.decode()
    return redirect(url)

@app.route('/user/autologin', methods=['GET'])
def auto_login():
    if 'key' not in request.args or len(request.args['key']) != 40:
        abort(403)
    code = str(request.args['key'][16:40])
    decryption_key = str(request.args['key'][0:16])
    the_key = 'da:auto_login:' + code
    info_text = r.get(the_key)
    if info_text is None:
        abort(403)
    r.delete(the_key)
    info_text = info_text.decode()
    try:
        info = decrypt_dictionary(info_text, decryption_key)
    except:
        abort(403)
    user = db.session.execute(select(UserModel).options(db.joinedload(UserModel.roles)).where(UserModel.id == info['user_id'])).scalar()
    if (not user) or user.social_id.startswith('disabled$'):
        abort(403)
    login_user(user, remember=False)
    update_last_login(user)
    if 'i' in info:
        url_info = dict(i=info['i'])
        if 'url_args' in info:
            url_info.update(info['url_args'])
        next_url = url_for('index', **url_info)
        if 'session' in info:
            update_session(info['i'], uid=info['session'], encrypted=info['encrypted'])
    elif 'next' in info:
        url_info = info.get('url_args', {})
        next_url = get_url_from_file_reference(info['next'], **url_info)
    else:
        next_url = url_for('interview_list', from_login='1')
    response = redirect(next_url)
    response.set_cookie('secret', info['secret'], httponly=True, secure=app.config['SESSION_COOKIE_SECURE'], samesite=app.config['SESSION_COOKIE_SAMESITE'])
    return response

@app.route('/headers', methods=['POST', 'GET'])
@csrf.exempt
def show_headers():
    return jsonify(headers=dict(request.headers), ipaddress=request.remote_addr)

@app.route('/authorize/<provider>', methods=['POST', 'GET'])
@csrf.exempt
def oauth_authorize(provider):
    if not current_user.is_anonymous:
        return redirect(url_for('interview_list', from_login='1'))
    oauth = OAuthSignIn.get_provider(provider)
    next_url = app.user_manager.make_safe_url_function(request.args.get('next', ''))
    if next_url:
        session['next'] = next_url
    return oauth.authorize()

@app.route('/callback/<provider>')
@csrf.exempt
def oauth_callback(provider):
    if not current_user.is_anonymous:
        return redirect(url_for('interview_list', from_login='1'))
    # for argument in request.args:
    #     logmessage("argument " + str(argument) + " is " + str(request.args[argument]))
    oauth = OAuthSignIn.get_provider(provider)
    social_id, username, email, name_data = oauth.callback()
    if social_id is None:
        flash(word('Authentication failed.'), 'error')
        return redirect(url_for('interview_list', from_login='1'))
    user = db.session.execute(select(UserModel).options(db.joinedload(UserModel.roles)).filter_by(social_id=social_id)).scalar()
    if not user:
        user = db.session.execute(select(UserModel).options(db.joinedload(UserModel.roles)).filter_by(email=email)).scalar()
    if user and user.social_id is not None and user.social_id.startswith('local'):
        flash(word('There is already a username and password on this system with the e-mail address') + " " + str(email) + ".  " + word("Please log in."), 'error')
        return redirect(url_for('user.login'))
    if not user:
        user = UserModel(social_id=social_id, nickname=username, email=email, active=True)
        if 'first_name' in name_data and 'last_name' in name_data and name_data['first_name'] is not None and name_data['last_name'] is not None:
            user.first_name = name_data['first_name']
            user.last_name = name_data['last_name']
        elif 'name' in name_data and name_data['name'] is not None and ' ' in name_data['name']:
            user.first_name = re.sub(r' .*', '', name_data['name'])
            user.last_name = re.sub(r'.* ', '', name_data['name'])
        db.session.add(user)
        db.session.commit()
    login_user(user, remember=False)
    update_last_login(user)
    if 'i' in session: #TEMPORARY
        get_session(session['i'])
    to_convert = []
    if 'tempuser' in session:
        to_convert.extend(sub_temp_user_dict_key(session['tempuser'], user.id))
    if 'sessions' in session:
        for filename, info in session['sessions'].items():
            if (filename, info['uid']) not in to_convert:
                to_convert.append((filename, info['uid']))
                save_user_dict_key(info['uid'], filename, priors=True, user=user)
                update_session(filename, key_logged=True)
    #logmessage("oauth_callback: calling substitute_secret")
    secret = substitute_secret(str(request.cookies.get('secret', None)), pad_to_16(MD5Hash(data=social_id).hexdigest()), to_convert=to_convert)
    sub_temp_other(user)
    if 'next' in session:
        the_url = session['next']
        del session['next']
        response = redirect(the_url)
    else:
        response = redirect(url_for('interview_list', from_login='1'))
    response.set_cookie('secret', secret, httponly=True, secure=app.config['SESSION_COOKIE_SECURE'], samesite=app.config['SESSION_COOKIE_SAMESITE'])
    return response

@app.route('/phone_login', methods=['POST', 'GET'])
def phone_login():
    if not app.config['USE_PHONE_LOGIN']:
        return ('File not found', 404)
    form = PhoneLoginForm(request.form)
    #next = request.args.get('next', url_for('interview_list'))
    if request.method == 'POST' and form.submit.data:
        ok = True
        if form.validate():
            phone_number = form.phone_number.data
            if docassemble.base.functions.phone_number_is_valid(phone_number):
                phone_number = docassemble.base.functions.phone_number_in_e164(phone_number)
            else:
                ok = False
        else:
            ok = False
        if ok:
            verification_code = random_digits(daconfig['verification code digits'])
            message = word("Your verification code is") + " " + str(verification_code) + "."
            user_agent = request.headers.get('User-Agent', '')
            if detect_mobile.search(user_agent):
                message += '  ' + word("You can also follow this link: ") + url_for('phone_login_verify', _external=True, p=phone_number, c=verification_code)
            tracker_prefix = 'da:phonelogin:ip:' + str(get_requester_ip(request)) + ':phone:'
            tracker_key = tracker_prefix + str(phone_number)
            pipe = r.pipeline()
            pipe.incr(tracker_key)
            pipe.expire(tracker_key, daconfig['ban period'])
            pipe.execute()
            total_attempts = 0
            for key in r.keys(tracker_prefix + '*'):
                val = r.get(key.decode())
                total_attempts += int(val)
            if total_attempts > daconfig['attempt limit']:
                logmessage("IP address " + str(get_requester_ip(request)) + " attempted to log in too many times.")
                flash(word("You have made too many login attempts."), 'error')
                return redirect(url_for('user.login'))
            total_attempts = 0
            for key in r.keys('da:phonelogin:ip:*:phone:' + phone_number):
                val = r.get(key.decode())
                total_attempts += int(val)
            if total_attempts > daconfig['attempt limit']:
                logmessage("Too many attempts were made to log in to phone number " + str(phone_number))
                flash(word("You have made too many login attempts."), 'error')
                return redirect(url_for('user.login'))
            key = 'da:phonelogin:' + str(phone_number) + ':code'
            pipe = r.pipeline()
            pipe.set(key, verification_code)
            pipe.expire(key, daconfig['verification code timeout'])
            pipe.execute()
            #logmessage("Writing code " + str(verification_code) + " to " + key)
            docassemble.base.functions.this_thread.current_info = current_info(req=request)
            success = docassemble.base.util.send_sms(to=phone_number, body=message)
            if success:
                session['phone_number'] = phone_number
                return redirect(url_for('phone_login_verify'))
            flash(word("There was a problem sending you a text message.  Please log in another way."), 'error')
            return redirect(url_for('user.login'))
        flash(word("Please enter a valid phone number"), 'error')
    return render_template('flask_user/phone_login.html', form=form, version_warning=None, title=word("Sign in with your mobile phone"), tab_title=word("Sign In"), page_title=word("Sign in"))

@app.route('/pv', methods=['POST', 'GET'])
def phone_login_verify():
    if not app.config['USE_PHONE_LOGIN']:
        return ('File not found', 404)
    phone_number = session.get('phone_number', request.args.get('p', None))
    if phone_number is None:
        return ('File not found', 404)
    form = PhoneLoginVerifyForm(request.form)
    form.phone_number.data = phone_number
    if 'c' in request.args and 'p' in request.args:
        submitted = True
        form.verification_code.data = request.args.get('c', None)
    else:
        submitted = False
    if submitted or (request.method == 'POST' and form.submit.data):
        if form.validate():
            social_id = 'phone$' + str(phone_number)
            user = db.session.execute(select(UserModel).options(db.joinedload(UserModel.roles)).filter_by(social_id=social_id)).scalar()
            if user and user.active is False:
                flash(word("Your account has been disabled."), 'error')
                return redirect(url_for('user.login'))
            if not user:
                user = UserModel(social_id=social_id, nickname=phone_number, active=True)
                db.session.add(user)
                db.session.commit()
            login_user(user, remember=False)
            update_last_login(user)
            r.delete('da:phonelogin:ip:' + str(get_requester_ip(request)) + ':phone:' + phone_number)
            to_convert = []
            if 'i' in session: #TEMPORARY
                get_session(session['i'])
            if 'tempuser' in session:
                to_convert.extend(sub_temp_user_dict_key(session['tempuser'], user.id))
            if 'sessions' in session:
                for filename, info in session['sessions'].items():
                    if (filename, info['uid']) not in to_convert:
                        to_convert.append((filename, info['uid']))
                        save_user_dict_key(info['uid'], filename, priors=True, user=user)
                        update_session(filename, key_logged=True)
            secret = substitute_secret(str(request.cookies.get('secret', None)), pad_to_16(MD5Hash(data=social_id).hexdigest()), to_convert=to_convert)
            response = redirect(url_for('interview_list', from_login='1'))
            response.set_cookie('secret', secret, httponly=True, secure=app.config['SESSION_COOKIE_SECURE'], samesite=app.config['SESSION_COOKIE_SAMESITE'])
            return response
        logmessage("IP address " + str(get_requester_ip(request)) + " made a failed login attempt using phone number " + str(phone_number) + ".")
        flash(word("Your verification code is invalid or expired.  Please try again."), 'error')
        return redirect(url_for('user.login'))
    return render_template('flask_user/phone_login_verify.html', form=form, version_warning=None, title=word("Verify your phone"), tab_title=word("Enter code"), page_title=word("Enter code"), description=word("We just sent you a text message with a verification code.  Enter the verification code to proceed."))

@app.route('/mfa_setup', methods=['POST', 'GET'])
def mfa_setup():
    in_login = False
    if current_user.is_authenticated:
        user = current_user
    elif 'validated_user' in session:
        in_login = True
        user = load_user(session['validated_user'])
    else:
        return ('File not found', 404)
    if not app.config['USE_MFA'] or not user.has_role(*app.config['MFA_ROLES']) or not user.social_id.startswith('local'):
        return ('File not found', 404)
    form = MFASetupForm(request.form)
    if request.method == 'POST' and form.submit.data:
        if 'otp_secret' not in session:
            return ('File not found', 404)
        otp_secret = session['otp_secret']
        del session['otp_secret']
        supplied_verification_code = re.sub(r'[^0-9]', '', form.verification_code.data)
        totp = pyotp.TOTP(otp_secret)
        if not totp.verify(supplied_verification_code):
            flash(word("Your verification code was invalid."), 'error')
            if in_login:
                del session['validated_user']
                if 'next' in session:
                    del session['next']
                return redirect(url_for('user.login'))
            return redirect(url_for('user_profile_page'))
        user = load_user(user.id)
        user.otp_secret = otp_secret
        db.session.commit()
        if in_login:
            if 'next' in session:
                next_url = session['next']
                del session['next']
            else:
                next_url = url_for('interview_list', from_login='1')
            return docassemble_flask_user.views._do_login_user(user, next_url, False)
        flash(word("You are now set up with two factor authentication."), 'success')
        return redirect(url_for('user_profile_page'))
    otp_secret = pyotp.random_base32()
    if user.email:
        the_name = user.email
    else:
        the_name = re.sub(r'.*\$', '', user.social_id)
    the_url = pyotp.totp.TOTP(otp_secret).provisioning_uri(the_name, issuer_name=app.config['APP_NAME'])
    im = qrcode.make(the_url, image_factory=qrcode.image.svg.SvgPathImage)
    output = BytesIO()
    im.save(output)
    the_qrcode = output.getvalue().decode()
    the_qrcode = re.sub(r"<\?xml version='1.0' encoding='UTF-8'\?>\n", '', the_qrcode)
    the_qrcode = re.sub(r'height="[0-9]+mm" ', '', the_qrcode)
    the_qrcode = re.sub(r'width="[0-9]+mm" ', '', the_qrcode)
    m = re.search(r'(viewBox="[^"]+")', the_qrcode)
    if m:
        viewbox = ' ' + m.group(1)
    else:
        viewbox = ''
    the_qrcode = '<svg class="damfasvg"' + viewbox + '><g transform="scale(1.0)">' + the_qrcode + '</g></svg>'
    session['otp_secret'] = otp_secret
    return render_template('flask_user/mfa_setup.html', form=form, version_warning=None, title=word("Two-factor authentication"), tab_title=word("Authentication"), page_title=word("Authentication"), description=word("Scan the barcode with your phone's authenticator app and enter the verification code."), the_qrcode=Markup(the_qrcode))

@login_required
@app.route('/mfa_reconfigure', methods=['POST', 'GET'])
def mfa_reconfigure():
    setup_translation()
    if not app.config['USE_MFA'] or not current_user.has_role(*app.config['MFA_ROLES']) or not current_user.social_id.startswith('local'):
        return ('File not found', 404)
    user = load_user(current_user.id)
    if user.otp_secret is None:
        if app.config['MFA_ALLOW_APP'] and (twilio_config is None or not app.config['MFA_ALLOW_SMS']):
            return redirect(url_for('mfa_setup'))
        if not app.config['MFA_ALLOW_APP']:
            return redirect(url_for('mfa_sms_setup'))
        return redirect(url_for('mfa_choose'))
    form = MFAReconfigureForm(request.form)
    if request.method == 'POST':
        if form.reconfigure.data:
            if app.config['MFA_ALLOW_APP'] and (twilio_config is None or not app.config['MFA_ALLOW_SMS']):
                return redirect(url_for('mfa_setup'))
            if not app.config['MFA_ALLOW_APP']:
                return redirect(url_for('mfa_sms_setup'))
            return redirect(url_for('mfa_choose'))
        if form.disable.data and not (len(app.config['MFA_REQUIRED_FOR_ROLE']) and current_user.has_role(*app.config['MFA_REQUIRED_FOR_ROLE'])):
            user.otp_secret = None
            db.session.commit()
            flash(word("Your account no longer uses two-factor authentication."), 'success')
            return redirect(url_for('user_profile_page'))
        if form.cancel.data:
            return redirect(url_for('user_profile_page'))
    if len(app.config['MFA_REQUIRED_FOR_ROLE']) > 0 and current_user.has_role(*app.config['MFA_REQUIRED_FOR_ROLE']):
        return render_template('flask_user/mfa_reconfigure.html', form=form, version_warning=None, title=word("Two-factor authentication"), tab_title=word("Authentication"), page_title=word("Authentication"), allow_disable=False, description=word("Would you like to reconfigure two-factor authentication?"))
    return render_template('flask_user/mfa_reconfigure.html', form=form, version_warning=None, title=word("Two-factor authentication"), tab_title=word("Authentication"), page_title=word("Authentication"), allow_disable=True, description=word("Your account already has two-factor authentication enabled.  Would you like to reconfigure or disable two-factor authentication?"))

@app.route('/mfa_choose', methods=['POST', 'GET'])
def mfa_choose():
    in_login = False
    if current_user.is_authenticated:
        user = current_user
    elif 'validated_user' in session:
        in_login = True
        user = load_user(session['validated_user'])
    else:
        return ('File not found', 404)
    if not app.config['USE_MFA'] or user.is_anonymous or not user.has_role(*app.config['MFA_ROLES']) or not user.social_id.startswith('local'):
        return ('File not found', 404)
    if app.config['MFA_ALLOW_APP'] and (twilio_config is None or not app.config['MFA_ALLOW_SMS']):
        return redirect(url_for('mfa_setup'))
    if not app.config['MFA_ALLOW_APP']:
        return redirect(url_for('mfa_sms_setup'))
    user = load_user(user.id)
    form = MFAChooseForm(request.form)
    if request.method == 'POST':
        if form.sms.data:
            return redirect(url_for('mfa_sms_setup'))
        if form.auth.data:
            return redirect(url_for('mfa_setup'))
        if in_login:
            del session['validated_user']
            if 'next' in session:
                del session['next']
            return redirect(url_for('user.login'))
        return redirect(url_for('user_profile_page'))
    return render_template('flask_user/mfa_choose.html', form=form, version_warning=None, title=word("Two-factor authentication"), tab_title=word("Authentication"), page_title=word("Authentication"), description=Markup(word("""Which type of two-factor authentication would you like to use?  The first option is to use an authentication app like <a target="_blank" href="https://en.wikipedia.org/wiki/Google_Authenticator">Google Authenticator</a> or <a target="_blank" href="https://authy.com/">Authy</a>.  The second option is to receive a text (SMS) message containing a verification code.""")))

@app.route('/mfa_sms_setup', methods=['POST', 'GET'])
def mfa_sms_setup():
    in_login = False
    if current_user.is_authenticated:
        user = current_user
    elif 'validated_user' in session:
        in_login = True
        user = load_user(session['validated_user'])
    else:
        return ('File not found', 404)
    if twilio_config is None or not app.config['USE_MFA'] or not user.has_role(*app.config['MFA_ROLES']) or not user.social_id.startswith('local'):
        return ('File not found', 404)
    form = MFASMSSetupForm(request.form)
    user = load_user(user.id)
    if request.method == 'GET' and user.otp_secret is not None and user.otp_secret.startswith(':phone:'):
        form.phone_number.data = re.sub(r'^:phone:', '', user.otp_secret)
    if request.method == 'POST' and form.submit.data:
        phone_number = form.phone_number.data
        if docassemble.base.functions.phone_number_is_valid(phone_number):
            phone_number = docassemble.base.functions.phone_number_in_e164(phone_number)
            verification_code = random_digits(daconfig['verification code digits'])
            message = word("Your verification code is") + " " + str(verification_code) + "."
            success = docassemble.base.util.send_sms(to=phone_number, body=message)
            if success:
                session['phone_number'] = phone_number
                key = 'da:mfa:phone:' + str(phone_number) + ':code'
                pipe = r.pipeline()
                pipe.set(key, verification_code)
                pipe.expire(key, daconfig['verification code timeout'])
                pipe.execute()
                return redirect(url_for('mfa_verify_sms_setup'))
            flash(word("There was a problem sending the text message."), 'error')
            if in_login:
                del session['validated_user']
                if 'next' in session:
                    del session['next']
                return redirect(url_for('user.login'))
            return redirect(url_for('user_profile_page'))
        flash(word("Invalid phone number."), 'error')
    return render_template('flask_user/mfa_sms_setup.html', form=form, version_warning=None, title=word("Two-factor authentication"), tab_title=word("Authentication"), page_title=word("Authentication"), description=word("""Enter your phone number.  A confirmation code will be sent to you."""))

@app.route('/mfa_verify_sms_setup', methods=['POST', 'GET'])
def mfa_verify_sms_setup():
    in_login = False
    if current_user.is_authenticated:
        user = current_user
    elif 'validated_user' in session:
        in_login = True
        user = load_user(session['validated_user'])
    else:
        return ('File not found', 404)
    if 'phone_number' not in session or twilio_config is None or not app.config['USE_MFA'] or not user.has_role(*app.config['MFA_ROLES']) or not user.social_id.startswith('local'):
        return ('File not found', 404)
    form = MFAVerifySMSSetupForm(request.form)
    if request.method == 'POST' and form.submit.data:
        phone_number = session['phone_number']
        del session['phone_number']
        key = 'da:mfa:phone:' + str(phone_number) + ':code'
        verification_code = r.get(key)
        r.delete(key)
        supplied_verification_code = re.sub(r'[^0-9]', '', form.verification_code.data)
        if verification_code is None:
            flash(word('Your verification code was missing or expired'), 'error')
            return redirect(url_for('user_profile_page'))
        if verification_code.decode() == supplied_verification_code:
            user = load_user(user.id)
            user.otp_secret = ':phone:' + phone_number
            db.session.commit()
            if in_login:
                if 'next' in session:
                    next_url = session['next']
                    del session['next']
                else:
                    next_url = url_for('interview_list', from_login='1')
                return docassemble_flask_user.views._do_login_user(user, next_url, False)
            flash(word("You are now set up with two factor authentication."), 'success')
            return redirect(url_for('user_profile_page'))
    return render_template('flask_user/mfa_verify_sms_setup.html', form=form, version_warning=None, title=word("Two-factor authentication"), tab_title=word("Authentication"), page_title=word("Authentication"), description=word('We just sent you a text message with a verification code.  Enter the verification code to proceed.'))

@app.route('/mfa_login', methods=['POST', 'GET'])
def mfa_login():
    if not app.config['USE_MFA']:
        logmessage("mfa_login: two factor authentication not configured")
        return ('File not found', 404)
    if 'validated_user' not in session:
        logmessage("mfa_login: validated_user not in session")
        return ('File not found', 404)
    user = load_user(session['validated_user'])
    if current_user.is_authenticated and current_user.id != user.id:
        del session['validated_user']
        return ('File not found', 404)
    if user is None or user.otp_secret is None or not user.social_id.startswith('local'):
        logmessage("mfa_login: user not setup for MFA where validated_user was " + str(session['validated_user']))
        return ('File not found', 404)
    form = MFALoginForm(request.form)
    if not form.next.data:
        form.next.data = _get_safe_next_param('next', url_for('interview_list', from_login='1'))
    if request.method == 'POST' and form.submit.data:
        del session['validated_user']
        if 'next' in session:
            safe_next = session['next']
            del session['next']
        else:
            safe_next = form.next.data
        fail_key = 'da:failedlogin:ip:' + str(get_requester_ip(request))
        failed_attempts = r.get(fail_key)
        if failed_attempts is not None and int(failed_attempts) > daconfig['attempt limit']:
            return ('File not found', 404)
        supplied_verification_code = re.sub(r'[^0-9]', '', form.verification_code.data)
        if user.otp_secret.startswith(':phone:'):
            phone_number = re.sub(r'^:phone:', '', user.otp_secret)
            key = 'da:mfa:phone:' + str(phone_number) + ':code'
            verification_code = r.get(key)
            r.delete(key)
            if verification_code is None or supplied_verification_code != verification_code.decode():
                r.incr(fail_key)
                r.expire(fail_key, 86400)
                flash(word("Your verification code was invalid or expired."), 'error')
                return redirect(url_for('user.login'))
            if failed_attempts is not None:
                r.delete(fail_key)
        else:
            totp = pyotp.TOTP(user.otp_secret)
            if not totp.verify(supplied_verification_code):
                r.incr(fail_key)
                r.expire(fail_key, 86400)
                flash(word("Your verification code was invalid."), 'error')
                if 'validated_user' in session:
                    del session['validated_user']
                if 'next' in session:
                    return redirect(url_for('user.login', next=session['next']))
                return redirect(url_for('user.login'))
            if failed_attempts is not None:
                r.delete(fail_key)
        return docassemble_flask_user.views._do_login_user(user, safe_next, False)
    description = word("This account uses two-factor authentication.")
    if user.otp_secret.startswith(':phone:'):
        description += "  " + word("Please enter the verification code from the text message we just sent you.")
    else:
        description += "  " + word("Please enter the verification code from your authentication app.")
    return render_template('flask_user/mfa_login.html', form=form, version_warning=None, title=word("Two-factor authentication"), tab_title=word("Authentication"), page_title=word("Authentication"), description=description)

@app.route('/user/manage', methods=['POST', 'GET'])
def manage_account():
    if (current_user.is_authenticated and current_user.has_roles(['admin'])) or not daconfig.get('user can delete account', True):
        abort(403)
    if current_user.is_anonymous and not daconfig.get('allow anonymous access', True):
        return redirect(url_for('user.login'))
    secret = request.cookies.get('secret', None)
    if current_user.is_anonymous:
        logged_in = False
        if 'tempuser' not in session:
            return ('File not found', 404)
        temp_user_id = int(session['tempuser'])
    else:
        logged_in = True
    delete_shared = daconfig.get('delete account deletes shared', False)
    form = ManageAccountForm(request.form)
    if request.method == 'POST' and form.validate():
        if current_user.is_authenticated:
            user_interviews(user_id=current_user.id, secret=secret, exclude_invalid=False, action='delete_all', delete_shared=delete_shared)
            the_user_id = current_user.id
            logout_user()
            delete_user_data(the_user_id, r, r_user)
        else:
            sessions_to_delete = set()
            interview_query = db.session.execute(select(UserDictKeys.filename, UserDictKeys.key).where(UserDictKeys.temp_user_id == temp_user_id).group_by(UserDictKeys.filename, UserDictKeys.key))
            for interview_info in interview_query:
                sessions_to_delete.add((interview_info.key, interview_info.filename))
            for session_id, yaml_filename in sessions_to_delete:
                manual_checkout(manual_session_id=session_id, manual_filename=yaml_filename)
                reset_user_dict(session_id, yaml_filename, temp_user_id=temp_user_id, force=delete_shared)
            delete_temp_user_data(temp_user_id, r)
        delete_session_info()
        session.clear()
        response = redirect(exit_page)
        response.set_cookie('remember_token', '', expires=0)
        response.set_cookie('visitor_secret', '', expires=0)
        response.set_cookie('secret', '', expires=0)
        response.set_cookie('session', '', expires=0)
        return response
    if logged_in:
        description = word("""You can delete your account on this page.  Type "delete my account" (in lowercase, without the quotes) into the box below and then press the "Delete account" button.  This will erase your interview sessions and your user profile.  To go back to your user profile page, press the "Cancel" button.""")
    else:
        description = word("""You can delete your account on this page.  Type "delete my account" (in lowercase, without the quotes) into the box below and then press the "Delete account" button.  This will erase your interview sessions.""")
    return render_template('pages/manage_account.html', form=form, version_warning=None, title=word("Manage account"), tab_title=word("Manage account"), page_title=word("Manage account"), description=description, logged_in=logged_in)

def get_github_flow():
    app_credentials = current_app.config['OAUTH_CREDENTIALS'].get('github', {})
    client_id = app_credentials.get('id', None)
    client_secret = app_credentials.get('secret', None)
    if client_id is None or client_secret is None:
        raise DAError('GitHub integration is not configured')
    flow = oauth2client.client.OAuth2WebServerFlow(
        client_id=client_id,
        client_secret=client_secret,
        scope='repo admin:public_key read:user user:email read:org',
        redirect_uri=url_for('github_oauth_callback', _external=True),
        auth_uri='http://github.com/login/oauth/authorize',
        token_uri='https://github.com/login/oauth/access_token',
        access_type='offline',
        prompt='consent')
    return flow

def delete_ssh_keys():
    area = SavedFile(current_user.id, fix=True, section='playgroundpackages')
    area.delete_file('.ssh-private')
    area.delete_file('.ssh-public')
    #area.delete_file('.ssh_command.sh')
    area.finalize()

def get_ssh_keys(email):
    area = SavedFile(current_user.id, fix=True, section='playgroundpackages')
    private_key_file = os.path.join(area.directory, '.ssh-private')
    public_key_file = os.path.join(area.directory, '.ssh-public')
    if (not (os.path.isfile(private_key_file) and os.path.isfile(private_key_file))) or (not (os.path.isfile(public_key_file) and os.path.isfile(public_key_file))):
        key = RSA.generate(4096)
        pubkey = key.publickey()
        area.write_content(key.exportKey('PEM').decode(), filename=private_key_file, save=False)
        pubkey_text = pubkey.exportKey('OpenSSH').decode() + " " + str(email) + "\n"
        area.write_content(pubkey_text, filename=public_key_file, save=False)
        area.finalize()
    return (private_key_file, public_key_file)

def get_next_link(resp):
    if 'link' in resp and resp['link']:
        link_info = links_from_header.extract(resp['link'])
        if 'next' in link_info:
            return link_info['next']
    return None

@app.route('/github_menu', methods=['POST', 'GET'])
@login_required
@roles_required(['admin', 'developer'])
def github_menu():
    if not app.config['ENABLE_PLAYGROUND']:
        return ('File not found', 404)
    if not app.config['USE_GITHUB']:
        return ('File not found', 404)
    setup_translation()
    form = GitHubForm(request.form)
    if request.method == 'POST':
        if form.configure.data:
            return redirect(url_for('github_configure'))
        if form.unconfigure.data:
            return redirect(url_for('github_unconfigure'))
        if form.cancel.data:
            return redirect(url_for('user_profile_page'))
        if form.save.data:
            info = {}
            info['shared'] = bool(form.shared.data)
            info['orgs'] = bool(form.orgs.data)
            r.set('da:using_github:userid:' + str(current_user.id), json.dumps(info))
            flash(word("Your GitHub settings were saved."), 'info')
    uses_github = r.get('da:using_github:userid:' + str(current_user.id))
    if uses_github is not None:
        uses_github = uses_github.decode()
        if uses_github == '1':
            form.shared.data = True
            form.orgs.data = True
        else:
            info = json.loads(uses_github)
            form.shared.data = info['shared']
            form.orgs.data = info['orgs']
        description = word("Your GitHub integration is currently turned on.  Below, you can change which repositories docassemble can access.  You can disable GitHub integration if you no longer wish to use it.")
    else:
        description = word("If you have a GitHub account, you can turn on GitHub integration.  This will allow you to use GitHub as a version control system for packages from inside the Playground.")
    return render_template('pages/github.html', form=form, version_warning=None, title=word("GitHub Integration"), tab_title=word("GitHub"), page_title=word("GitHub"), description=description, uses_github=uses_github, bodyclass='daadminbody')

@app.route('/github_configure', methods=['POST', 'GET'])
@login_required
@roles_required(['admin', 'developer'])
def github_configure():
    if not app.config['ENABLE_PLAYGROUND']:
        return ('File not found', 404)
    if not app.config['USE_GITHUB']:
        return ('File not found', 404)
    setup_translation()
    storage = RedisCredStorage(app='github')
    credentials = storage.get()
    if not credentials or credentials.invalid:
        state_string = random_string(16)
        session['github_next'] = json.dumps(dict(state=state_string, path='github_configure', arguments=request.args))
        flow = get_github_flow()
        uri = flow.step1_get_authorize_url(state=state_string)
        return redirect(uri)
    http = credentials.authorize(httplib2.Http())
    found = False
    resp, content = http.request("https://api.github.com/user/emails", "GET")
    if int(resp['status']) == 200:
        user_info_list = json.loads(content.decode())
        user_info = None
        for item in user_info_list:
            if item.get('email', None) and item.get('visibility', None) != 'private':
                user_info = item
        if user_info is None:
            raise DAError("github_configure: could not get e-mail address")
    else:
        raise DAError("github_configure: could not get information about user")
    resp, content = http.request("https://api.github.com/user/keys", "GET")
    if int(resp['status']) == 200:
        for key in json.loads(content.decode()):
            if key['title'] == app.config['APP_NAME'] or key['title'] == app.config['APP_NAME'] + '_user_' + str(current_user.id):
                found = True
    else:
        raise DAError("github_configure: could not get information about ssh keys")
    while found is False:
        next_link = get_next_link(resp)
        if next_link:
            resp, content = http.request(next_link, "GET")
            if int(resp['status']) == 200:
                for key in json.loads(content.decode()):
                    if key['title'] == app.config['APP_NAME'] or key['title'] == app.config['APP_NAME'] + '_user_' + str(current_user.id):
                        found = True
            else:
                raise DAError("github_configure: could not get additional information about ssh keys")
        else:
            break
    if found:
        flash(word("Your GitHub integration has already been configured."), 'info')
    if not found:
        (private_key_file, public_key_file) = get_ssh_keys(user_info['email'])
        with open(public_key_file, 'r', encoding='utf-8') as fp:
            public_key = fp.read()
        headers = {'Content-Type': 'application/json'}
        body = json.dumps(dict(title=app.config['APP_NAME'] + '_user_' + str(current_user.id), key=public_key))
        resp, content = http.request("https://api.github.com/user/keys", "POST", headers=headers, body=body)
        if int(resp['status']) == 201:
            flash(word("GitHub integration was successfully configured."), 'info')
        else:
            raise DAError("github_configure: error setting public key")
    r.set('da:using_github:userid:' + str(current_user.id), json.dumps(dict(shared=True, orgs=True)))
    return redirect(url_for('github_menu'))

@app.route('/github_unconfigure', methods=['POST', 'GET'])
@login_required
@roles_required(['admin', 'developer'])
def github_unconfigure():
    if not app.config['ENABLE_PLAYGROUND']:
        return ('File not found', 404)
    if not app.config['USE_GITHUB']:
        return ('File not found', 404)
    setup_translation()
    storage = RedisCredStorage(app='github')
    credentials = storage.get()
    if not credentials or credentials.invalid:
        state_string = random_string(16)
        session['github_next'] = json.dumps(dict(state=state_string, path='github_unconfigure', arguments=request.args))
        flow = get_github_flow()
        uri = flow.step1_get_authorize_url(state=state_string)
        return redirect(uri)
    http = credentials.authorize(httplib2.Http())
    ids_to_remove = []
    try:
        resp, content = http.request("https://api.github.com/user/keys", "GET")
        if int(resp['status']) == 200:
            for key in json.loads(content.decode()):
                if key['title'] == app.config['APP_NAME'] or key['title'] == app.config['APP_NAME'] + '_user_' + str(current_user.id):
                    ids_to_remove.append(key['id'])
        else:
            raise DAError("github_configure: could not get information about ssh keys")
        while True:
            next_link = get_next_link(resp)
            if next_link:
                resp, content = http.request(next_link, "GET")
                if int(resp['status']) == 200:
                    for key in json.loads(content.decode()):
                        if key['title'] == app.config['APP_NAME'] or key['title'] == app.config['APP_NAME'] + '_user_' + str(current_user.id):
                            ids_to_remove.append(key['id'])
                else:
                    raise DAError("github_unconfigure: could not get additional information about ssh keys")
            else:
                break
        for id_to_remove in ids_to_remove:
            resp, content = http.request("https://api.github.com/user/keys/" + str(id_to_remove), "DELETE")
            if int(resp['status']) != 204:
                raise DAError("github_unconfigure: error deleting public key " + str(id_to_remove) + ": " + str(resp['status']) + " content: " + content.decode())
    except:
        logmessage("Error deleting SSH keys on GitHub")
    delete_ssh_keys()
    r.delete('da:github:userid:' + str(current_user.id))
    r.delete('da:using_github:userid:' + str(current_user.id))
    flash(word("GitHub integration was successfully disconnected."), 'info')
    return redirect(url_for('user_profile_page'))

@app.route('/github_oauth_callback', methods=['POST', 'GET'])
@login_required
@roles_required(['admin', 'developer'])
def github_oauth_callback():
    if not app.config['ENABLE_PLAYGROUND']:
        return ('File not found', 404)
    setup_translation()
    failed = False
    if not app.config['USE_GITHUB']:
        logmessage('github_oauth_callback: server does not use github')
        failed = True
    elif 'github_next' not in session:
        logmessage('github_oauth_callback: next not in session')
        failed = True
    if failed is False:
        github_next = json.loads(session['github_next'])
        del session['github_next']
        if 'code' not in request.args or 'state' not in request.args:
            logmessage('github_oauth_callback: code and state not in args')
            failed = True
        elif request.args['state'] != github_next['state']:
            logmessage('github_oauth_callback: state did not match')
            failed = True
    if failed:
        r.delete('da:github:userid:' + str(current_user.id))
        r.delete('da:using_github:userid:' + str(current_user.id))
        return ('File not found', 404)
    flow = get_github_flow()
    credentials = flow.step2_exchange(request.args['code'])
    storage = RedisCredStorage(app='github')
    storage.put(credentials)
    return redirect(github_next['path'], **github_next['arguments'])

@app.route('/user/google-sign-in')
def google_page():
    return render_template('flask_user/google_login.html', version_warning=None, title=word("Sign In"), tab_title=word("Sign In"), page_title=word("Sign in"))

@app.route("/user/post-sign-in", methods=['GET'])
def post_sign_in():
    return redirect(url_for('interview_list', from_login='1'))

@app.route("/leave", methods=['GET'])
def leave():
    if 'language' in session:
        the_exit_page = request.args.get('next', exit_page)
    else:
        the_exit_page = exit_page
    # if current_user.is_authenticated:
    #     flask_user.signals.user_logged_out.send(current_app._get_current_object(), user=current_user)
    #     logout_user()
    #delete_session_for_interview(i=request.args.get('i', None))
    #delete_session_info()
    #response = redirect(exit_page)
    #response.set_cookie('remember_token', '', expires=0)
    #response.set_cookie('visitor_secret', '', expires=0)
    #response.set_cookie('secret', '', expires=0)
    #response.set_cookie('session', '', expires=0)
    #return response
    return redirect(the_exit_page)

@app.route("/restart_session", methods=['GET'])
def restart_session():
    yaml_filename = request.args.get('i', None)
    if yaml_filename is None:
        return redirect(url_for('index'))
    session_info = get_session(yaml_filename)
    if session_info is None:
        return redirect(url_for('index'))
    session_id = session_info['uid']
    manual_checkout(manual_filename=yaml_filename)
    if 'visitor_secret' in request.cookies:
        secret = request.cookies['visitor_secret']
    else:
        secret = request.cookies.get('secret', None)
    if secret is not None:
        secret = str(secret)
    try:
        steps, user_dict, is_encrypted = fetch_user_dict(session_id, yaml_filename, secret=secret)
    except:
        return redirect(url_for('index', i=yaml_filename))
    url_args = user_dict['url_args']
    url_args['reset'] = '1'
    url_args['i'] = yaml_filename
    return redirect(url_for('index', **url_args))

@app.route("/new_session", methods=['GET'])
def new_session():
    yaml_filename = request.args.get('i', None)
    if yaml_filename is None:
        return redirect(url_for('index'))
    manual_checkout(manual_filename=yaml_filename)
    url_args = dict(i=yaml_filename, new_session='1')
    return redirect(url_for('index', **url_args))

@app.route("/exit", methods=['GET'])
def exit_endpoint():
    if 'language' in session:
        the_exit_page = request.args.get('next', exit_page)
    else:
        the_exit_page = exit_page
    yaml_filename = request.args.get('i', None)
    if yaml_filename is not None:
        session_info = get_session(yaml_filename)
        if session_info is not None:
            manual_checkout(manual_filename=yaml_filename)
            reset_user_dict(session_info['uid'], yaml_filename)
    delete_session_for_interview(i=yaml_filename)
    return redirect(the_exit_page)

@app.route("/exit_logout", methods=['GET'])
def exit_logout():
    if 'language' in session:
        the_exit_page = request.args.get('next', exit_page)
    else:
        the_exit_page = exit_page
    yaml_filename = request.args.get('i', guess_yaml_filename())
    if yaml_filename is not None:
        session_info = get_session(yaml_filename)
        if session_info is not None:
            manual_checkout(manual_filename=yaml_filename)
            reset_user_dict(session_info['uid'], yaml_filename)
    if current_user.is_authenticated:
        docassemble_flask_user.signals.user_logged_out.send(current_app._get_current_object(), user=current_user)
        logout_user()
    session.clear()
    response = redirect(the_exit_page)
    response.set_cookie('remember_token', '', expires=0)
    response.set_cookie('visitor_secret', '', expires=0)
    response.set_cookie('secret', '', expires=0)
    response.set_cookie('session', '', expires=0)
    return response

@app.route("/cleanup_sessions", methods=['GET'])
def cleanup_sessions():
    kv_session.cleanup_sessions()
    return render_template('base_templates/blank.html')

ready_file = os.path.join(os.path.dirname(WEBAPP_PATH), 'ready')

@app.route("/health_status", methods=['GET'])
def health_status():
    ok = True
    if request.args.get('ready', False):
        if not os.path.isfile(ready_file):
            ok = False
    return jsonify({'ok': ok, 'server_start_time': START_TIME, 'version': da_version})

@app.route("/health_check", methods=['GET'])
def health_check():
    if request.args.get('ready', False):
        if not os.path.isfile(ready_file):
            return ('', 400)
    response = make_response(render_template('pages/health_check.html', content="OK"), 200)
    response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
    return response

@app.route("/checkout", methods=['POST'])
def checkout():
    try:
        manual_checkout(manual_filename=request.args['i'])
    except:
        return jsonify(success=False)
    return jsonify(success=True)

@app.route("/restart_ajax", methods=['POST'])
@login_required
@roles_required(['admin', 'developer'])
def restart_ajax():
    #logmessage("restart_ajax: action is " + str(request.form.get('action', None)))
    #if current_user.has_role('admin', 'developer'):
    #    logmessage("restart_ajax: user has permission")
    #else:
    #    logmessage("restart_ajax: user has no permission")
    if request.form.get('action', None) == 'restart' and current_user.has_role('admin', 'developer'):
        logmessage("restart_ajax: restarting")
        restart_all()
        return jsonify(success=True)
    return jsonify(success=False)

class ChatPartners:
    pass

def get_current_chat_log(yaml_filename, session_id, secret, utc=True, timezone=None):
    if timezone is None:
        timezone = get_default_timezone()
    timezone = pytz.timezone(timezone)
    output = []
    if yaml_filename is None or session_id is None:
        return output
    user_cache = {}
    for record in db.session.execute(select(ChatLog).where(and_(ChatLog.filename == yaml_filename, ChatLog.key == session_id)).order_by(ChatLog.id)).scalars():
        if record.encrypted:
            try:
                message = decrypt_phrase(record.message, secret)
            except:
                sys.stderr.write("get_current_chat_log: Could not decrypt phrase with secret " + secret + "\n")
                continue
        else:
            message = unpack_phrase(record.message)
        # if record.temp_owner_id:
        #     owner_first_name = None
        #     owner_last_name = None
        #     owner_email = None
        # elif record.owner_id in user_cache:
        #     owner_first_name = user_cache[record.owner_id].first_name
        #     owner_last_name = user_cache[record.owner_id].last_name
        #     owner_email = user_cache[record.owner_id].email
        # else:
        #     sys.stderr.write("get_current_chat_log: Invalid owner ID in chat log\n")
        #     continue
        if record.temp_user_id:
            user_first_name = None
            user_last_name = None
            user_email = None
        elif record.user_id in user_cache:
            user_first_name = user_cache[record.user_id].first_name
            user_last_name = user_cache[record.user_id].last_name
            user_email = user_cache[record.user_id].email
        else:
            new_user = get_user_object(record.user_id)
            if new_user is None:
                sys.stderr.write("get_current_chat_log: Invalid user ID in chat log\n")
                continue
            user_cache[record.user_id] = new_user
            user_first_name = user_cache[record.user_id].first_name
            user_last_name = user_cache[record.user_id].last_name
            user_email = user_cache[record.user_id].email
        if utc:
            the_datetime = record.modtime.replace(tzinfo=tz.tzutc())
        else:
            the_datetime = record.modtime.replace(tzinfo=tz.tzutc()).astimezone(timezone)
        output.append(dict(message=message, datetime=the_datetime, user_email=user_email, user_first_name=user_first_name, user_last_name=user_last_name))
    return output

def jsonify_with_cache(*pargs, **kwargs):
    response = jsonify(*pargs, **kwargs)
    response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
    return response

@app.route("/checkin", methods=['POST', 'GET'])
def checkin():
    yaml_filename = request.args.get('i', None)
    if yaml_filename is None:
        return jsonify_with_cache(success=False)
    session_info = get_session(yaml_filename)
    if session_info is None:
        return jsonify_with_cache(success=False)
    session_id = session_info['uid']
    if 'visitor_secret' in request.cookies:
        secret = request.cookies['visitor_secret']
    else:
        secret = request.cookies.get('secret', None)
    if secret is not None:
        secret = str(secret)
    if current_user.is_anonymous:
        if 'tempuser' not in session:
            return jsonify_with_cache(success=False)
        the_user_id = 't' + str(session['tempuser'])
        auth_user_id = None
        temp_user_id = int(session['tempuser'])
    else:
        auth_user_id = current_user.id
        the_user_id = current_user.id
        temp_user_id = None
    if request.form.get('action', None) == 'chat_log':
        #sys.stderr.write("checkin: fetch_user_dict1\n")
        steps, user_dict, is_encrypted = fetch_user_dict(session_id, yaml_filename, secret=secret)
        if user_dict is None or user_dict['_internal']['livehelp']['availability'] != 'available':
            return jsonify_with_cache(success=False)
        messages = get_chat_log(user_dict['_internal']['livehelp']['mode'], yaml_filename, session_id, auth_user_id, temp_user_id, secret, auth_user_id, temp_user_id)
        return jsonify_with_cache(success=True, messages=messages)
    if request.form.get('action', None) == 'checkin':
        commands = []
        checkin_code = request.form.get('checkinCode', None)
        do_action = request.form.get('do_action', None)
        #logmessage("in checkin")
        if do_action is not None:
            parameters = {}
            form_parameters = request.form.get('parameters', None)
            if form_parameters is not None:
                parameters = json.loads(form_parameters)
            #logmessage("Action was " + str(do_action) + " and parameters were " + repr(parameters))
            obtain_lock(session_id, yaml_filename)
            #sys.stderr.write("checkin: fetch_user_dict2\n")
            steps, user_dict, is_encrypted = fetch_user_dict(session_id, yaml_filename, secret=secret)
            interview = docassemble.base.interview_cache.get_interview(yaml_filename)
            interview_status = docassemble.base.parse.InterviewStatus(current_info=current_info(yaml=yaml_filename, req=request, action=None, session_info=session_info, secret=secret, device_id=request.cookies.get('ds', None)))
            interview_status.checkin = True
            interview.assemble(user_dict, interview_status=interview_status)
            interview_status.current_info.update(dict(action=do_action, arguments=parameters))
            interview.assemble(user_dict, interview_status=interview_status)
            if interview_status.question.question_type == "backgroundresponse":
                the_response = interview_status.question.backgroundresponse
                if isinstance(the_response, dict) and 'pargs' in the_response and isinstance(the_response['pargs'], list) and len(the_response['pargs']) == 2 and the_response['pargs'][1] in ('javascript', 'flash', 'refresh', 'fields'):
                    commands.append(dict(action=do_action, value=docassemble.base.functions.safe_json(the_response['pargs'][0]), extra=the_response['pargs'][1]))
                elif isinstance(the_response, list) and len(the_response) == 2 and the_response[1] in ('javascript', 'flash', 'refresh', 'fields'):
                    commands.append(dict(action=do_action, value=docassemble.base.functions.safe_json(the_response[0]), extra=the_response[1]))
                elif isinstance(the_response, str) and the_response == 'refresh':
                    commands.append(dict(action=do_action, value=docassemble.base.functions.safe_json(None), extra='refresh'))
                else:
                    commands.append(dict(action=do_action, value=docassemble.base.functions.safe_json(the_response), extra='backgroundresponse'))
            elif interview_status.question.question_type == "template" and interview_status.question.target is not None:
                commands.append(dict(action=do_action, value=dict(target=interview_status.question.target, content=docassemble.base.util.markdown_to_html(interview_status.questionText, trim=True)), extra='backgroundresponse'))
            save_user_dict(session_id, user_dict, yaml_filename, secret=secret, encrypt=is_encrypted, steps=steps)
            release_lock(session_id, yaml_filename)
        peer_ok = False
        help_ok = False
        num_peers = 0
        help_available = 0
        session_info = get_session(yaml_filename)
        old_chatstatus = session_info['chatstatus']
        chatstatus = request.form.get('chatstatus', 'off')
        if old_chatstatus != chatstatus:
            update_session(yaml_filename, chatstatus=chatstatus)
        obj = dict(chatstatus=chatstatus, i=yaml_filename, uid=session_id, userid=the_user_id)
        key = 'da:session:uid:' + str(session_id) + ':i:' + str(yaml_filename) + ':userid:' + str(the_user_id)
        call_forwarding_on = False
        forwarding_phone_number = None
        if twilio_config is not None:
            forwarding_phone_number = twilio_config['name']['default'].get('number', None)
            if forwarding_phone_number is not None:
                call_forwarding_on = True
        call_forwarding_code = None
        call_forwarding_message = None
        if call_forwarding_on:
            for call_key in r.keys(re.sub(r'^da:session:uid:', 'da:phonecode:monitor:*:uid:', key)):
                call_key = call_key.decode()
                call_forwarding_code = r.get(call_key)
                if call_forwarding_code is not None:
                    call_forwarding_code = call_forwarding_code.decode()
                    other_value = r.get('da:callforward:' + call_forwarding_code)
                    if other_value is None:
                        r.delete(call_key)
                        continue
                    other_value = other_value.decode()
                    remaining_seconds = r.ttl(call_key)
                    if remaining_seconds > 30:
                        call_forwarding_message = '<span class="daphone-message"><i class="fas fa-phone"></i> ' + word('To reach an advocate who can assist you, call') + ' <a class="daphone-number" href="tel:' + str(forwarding_phone_number) + '">' + str(forwarding_phone_number) + '</a> ' + word("and enter the code") + ' <span class="daphone-code">' + str(call_forwarding_code) + '</span>.</span>'
                        break
        chat_session_key = 'da:interviewsession:uid:' + str(session_id) + ':i:' + str(yaml_filename) + ':userid:' + str(the_user_id)
        potential_partners = []
        if str(chatstatus) != 'off': #in ('waiting', 'standby', 'ringing', 'ready', 'on', 'hangup', 'observeonly'):
            #sys.stderr.write("checkin: fetch_user_dict3\n")
            steps, user_dict, is_encrypted = fetch_user_dict(session_id, yaml_filename, secret=secret)
            if user_dict is None:
                sys.stderr.write("checkin: error accessing dictionary for %s and %s" % (session_id, yaml_filename))
                return jsonify_with_cache(success=False)
            obj['chatstatus'] = chatstatus
            obj['secret'] = secret
            obj['encrypted'] = is_encrypted
            obj['mode'] = user_dict['_internal']['livehelp']['mode']
            if obj['mode'] in ('peer', 'peerhelp'):
                peer_ok = True
            if obj['mode'] in ('help', 'peerhelp'):
                help_ok = True
            obj['partner_roles'] = user_dict['_internal']['livehelp']['partner_roles']
            if current_user.is_authenticated:
                for attribute in ('email', 'confirmed_at', 'first_name', 'last_name', 'country', 'subdivisionfirst', 'subdivisionsecond', 'subdivisionthird', 'organization', 'timezone', 'language'):
                    obj[attribute] = str(getattr(current_user, attribute, None))
            else:
                obj['temp_user_id'] = temp_user_id
            if help_ok and len(obj['partner_roles']) and not r.exists('da:block:uid:' + str(session_id) + ':i:' + str(yaml_filename) + ':userid:' + str(the_user_id)):
                pipe = r.pipeline()
                for role in obj['partner_roles']:
                    role_key = 'da:chat:roletype:' + str(role)
                    pipe.set(role_key, 1)
                    pipe.expire(role_key, 2592000)
                pipe.execute()
                for role in obj['partner_roles']:
                    for the_key in r.keys('da:monitor:role:' + role + ':userid:*'):
                        user_id = re.sub(r'^.*:userid:', '', the_key.decode())
                        if user_id not in potential_partners:
                            potential_partners.append(user_id)
                for the_key in r.keys('da:monitor:chatpartners:*'):
                    user_id = re.sub(r'^.*chatpartners:', '', the_key.decode())
                    if user_id not in potential_partners:
                        for chat_key in r.hgetall(the_key):
                            if chat_key.decode() == chat_session_key:
                                potential_partners.append(user_id)
            if len(potential_partners) > 0:
                if chatstatus == 'ringing':
                    lkey = 'da:ready:uid:' + str(session_id) + ':i:' + str(yaml_filename) + ':userid:' + str(the_user_id)
                    #logmessage("Writing to " + str(lkey))
                    pipe = r.pipeline()
                    failure = True
                    for user_id in potential_partners:
                        for the_key in r.keys('da:monitor:available:' + str(user_id)):
                            pipe.rpush(lkey, the_key.decode())
                            failure = False
                    if peer_ok:
                        for the_key in r.keys('da:interviewsession:uid:' + str(session_id) + ':i:' + str(yaml_filename) + ':userid:*'):
                            the_key = the_key.decode()
                            if the_key != chat_session_key:
                                pipe.rpush(lkey, the_key)
                                failure = False
                    if failure:
                        if peer_ok:
                            chatstatus = 'ready'
                        else:
                            chatstatus = 'waiting'
                        update_session(yaml_filename, chatstatus=chatstatus)
                        obj['chatstatus'] = chatstatus
                    else:
                        pipe.expire(lkey, 60)
                        pipe.execute()
                        chatstatus = 'ready'
                        update_session(yaml_filename, chatstatus=chatstatus)
                        obj['chatstatus'] = chatstatus
                elif chatstatus == 'on':
                    if len(potential_partners) > 0:
                        already_connected_to_help = False
                        for user_id in potential_partners:
                            for the_key in r.hgetall('da:monitor:chatpartners:' + str(user_id)):
                                if the_key.decode() == chat_session_key:
                                    already_connected_to_help = True
                        if not already_connected_to_help:
                            for user_id in potential_partners:
                                mon_sid = r.get('da:monitor:available:' + str(user_id))
                                if mon_sid is None:
                                    continue
                                mon_sid = mon_sid.decode()
                                int_sid = r.get('da:interviewsession:uid:' + str(session_id) + ':i:' + str(yaml_filename) + ':userid:' + str(the_user_id))
                                if int_sid is None:
                                    continue
                                int_sid = int_sid.decode()
                                r.publish(mon_sid, json.dumps(dict(messagetype='chatready', uid=session_id, i=yaml_filename, userid=the_user_id, secret=secret, sid=int_sid)))
                                r.publish(int_sid, json.dumps(dict(messagetype='chatpartner', sid=mon_sid)))
                                break
                if chatstatus in ('waiting', 'hangup'):
                    chatstatus = 'standby'
                    update_session(yaml_filename, chatstatus=chatstatus)
                    obj['chatstatus'] = chatstatus
            else:
                if peer_ok:
                    if chatstatus == 'ringing':
                        lkey = 'da:ready:uid:' + str(session_id) + ':i:' + str(yaml_filename) + ':userid:' + str(the_user_id)
                        pipe = r.pipeline()
                        failure = True
                        for the_key in r.keys('da:interviewsession:uid:' + str(session_id) + ':i:' + str(yaml_filename) + ':userid:*'):
                            the_key = the_key.decode()
                            if the_key != chat_session_key:
                                pipe.rpush(lkey, the_key)
                                failure = False
                        if not failure:
                            pipe.expire(lkey, 6000)
                            pipe.execute()
                        chatstatus = 'ready'
                        update_session(yaml_filename, chatstatus=chatstatus)
                        obj['chatstatus'] = chatstatus
                    elif chatstatus in ('waiting', 'hangup'):
                        chatstatus = 'standby'
                        update_session(yaml_filename, chatstatus=chatstatus)
                        obj['chatstatus'] = chatstatus
                else:
                    if chatstatus in ('standby', 'ready', 'ringing', 'hangup'):
                        chatstatus = 'waiting'
                        update_session(yaml_filename, chatstatus=chatstatus)
                        obj['chatstatus'] = chatstatus
            if peer_ok:
                for sess_key in r.keys('da:session:uid:' + str(session_id) + ':i:' + str(yaml_filename) + ':userid:*'):
                    if sess_key.decode() != key:
                        num_peers += 1
        help_available = len(potential_partners)
        html_key = 'da:html:uid:' + str(session_id) + ':i:' + str(yaml_filename) + ':userid:' + str(the_user_id)
        if old_chatstatus != chatstatus:
            html = r.get(html_key)
            if html is not None:
                html_obj = json.loads(html.decode())
                if 'browser_title' in html_obj:
                    obj['browser_title'] = html_obj['browser_title']
                obj['blocked'] = bool(r.exists('da:block:uid:' + str(session_id) + ':i:' + str(yaml_filename) + ':userid:' + str(the_user_id)))
                r.publish('da:monitor', json.dumps(dict(messagetype='sessionupdate', key=key, session=obj)))
            else:
                logmessage("checkin: the html was not found at " + str(html_key))
        pipe = r.pipeline()
        pipe.set(key, pickle.dumps(obj))
        pipe.expire(key, 60)
        pipe.expire(html_key, 60)
        pipe.execute()
        ocontrol_key = 'da:control:uid:' + str(session_id) + ':i:' + str(yaml_filename) + ':userid:' + str(the_user_id)
        ocontrol = r.get(ocontrol_key)
        observer_control = not bool(ocontrol is None)
        parameters = request.form.get('raw_parameters', None)
        if parameters is not None:
            key = 'da:input:uid:' + str(session_id) + ':i:' + str(yaml_filename) + ':userid:' + str(the_user_id)
            r.publish(key, parameters)
        worker_key = 'da:worker:uid:' + str(session_id) + ':i:' + str(yaml_filename) + ':userid:' + str(the_user_id)
        worker_len = r.llen(worker_key)
        if worker_len > 0:
            workers_inspected = 0
            while workers_inspected <= worker_len:
                worker_id = r.lpop(worker_key)
                if worker_id is not None:
                    try:
                        result = docassemble.webapp.worker.workerapp.AsyncResult(id=worker_id)
                        if result.ready():
                            if isinstance(result.result, ReturnValue):
                                commands.append(dict(value=result.result.value, extra=result.result.extra))
                        else:
                            r.rpush(worker_key, worker_id)
                    except Exception as errstr:
                        logmessage("checkin: got error " + str(errstr))
                        r.rpush(worker_key, worker_id)
                workers_inspected += 1
        if peer_ok or help_ok:
            return jsonify_with_cache(success=True, chat_status=chatstatus, num_peers=num_peers, help_available=help_available, phone=call_forwarding_message, observerControl=observer_control, commands=commands, checkin_code=checkin_code)
        return jsonify_with_cache(success=True, chat_status=chatstatus, phone=call_forwarding_message, observerControl=observer_control, commands=commands, checkin_code=checkin_code)
    return jsonify_with_cache(success=False)

@app.before_first_request
def setup_celery():
    docassemble.webapp.worker.workerapp.set_current()

@app.before_request
def setup_variables():
    #sys.stderr.write("Request on " + str(os.getpid()) + " " + str(threading.current_thread().ident) + " for " + request.path + " at " + time.strftime("%Y-%m-%d %H:%M:%S") + "\n")
    #g.request_start_time = time.time()
    #docassemble.base.functions.reset_thread_variables()
    docassemble.base.functions.reset_local_variables()

@app.after_request
def apply_security_headers(response):
    if app.config['SESSION_COOKIE_SECURE']:
        response.headers['Strict-Transport-Security'] = 'max-age=31536000'
    if 'embed' in g:
        return response
    response.headers["X-Content-Type-Options"] = 'nosniff'
    response.headers["X-XSS-Protection"] = '1'
    if daconfig.get('allow embedding', False) is not True:
        response.headers["X-Frame-Options"] = 'SAMEORIGIN'
        response.headers["Content-Security-Policy"] = "frame-ancestors 'self';"
    elif daconfig.get('cross site domains', []):
        response.headers["Content-Security-Policy"] = "frame-ancestors 'self' " + ' '.join(daconfig['cross site domains']) + ';'
    return response

# @app.after_request
# def print_time_of_request(response):
#     time_spent = time.time() - g.request_start_time
#     sys.stderr.write("Request on " + str(os.getpid()) + " " + str(threading.current_thread().ident) + " complete after " + str("%.5fs" % time_spent) + "\n")
#     if time_spent > 3.0:
#         if hasattr(g, 'start_index'):
#             logmessage("Duration to beginning: %fs" % (g.start_index - g.request_start_time))
#         if hasattr(g, 'got_dict'):
#             logmessage("Duration to getting dictionary: %fs" % (g.got_dict - g.request_start_time))
#         if hasattr(g, 'before_interview'):
#             logmessage("Duration to before interview: %fs" % (g.before_interview - g.request_start_time))
#         if hasattr(g, 'after_interview'):
#             logmessage("Duration to after interview: %fs" % (g.after_interview - g.request_start_time))
#         if hasattr(g, 'status_created'):
#             logmessage("Duration to status: %fs" % (g.status_created - g.request_start_time))
#         if hasattr(g, 'assembly_start'):
#             logmessage("Duration to assembly start: %fs" % (g.assembly_start - g.request_start_time))
#         if hasattr(g, 'assembly_end'):
#             logmessage("Duration to assembly end: %fs" % (g.assembly_end - g.request_start_time))
#         logmessage("Duration to end of request: %fs" % time_spent)
#         if hasattr(g, 'interview') and hasattr(g, 'interview_status'):
#             logmessage(to_text(get_history(g.interview, g.interview_status)))
#     return response

# @app.before_request
# def setup_celery():
#     docassemble.webapp.worker.workerapp.set_current()

# @app.before_request
# def before_request():
#     docassemble.base.functions.reset_thread_variables()
#     docassemble.base.functions.reset_local_variables()
#     g.request_start_time = time.time()
#     g.request_time = lambda: "%.5fs" % (time.time() - g.request_start_time)

@app.route("/vars", methods=['POST', 'GET'])
def get_variables():
    yaml_filename = request.args.get('i', None)
    if yaml_filename is None:
        return ("Invalid request", 400)
    session_info = get_session(yaml_filename)
    if session_info is None:
        return ("Invalid request", 400)
    session_id = session_info['uid']
    if 'visitor_secret' in request.cookies:
        secret = request.cookies['visitor_secret']
    else:
        secret = request.cookies.get('secret', None)
    if secret is not None:
        secret = str(secret)
    #session_cookie_id = request.cookies.get('session', None)
    if session_id is None or yaml_filename is None:
        return jsonify(success=False)
    #sys.stderr.write("get_variables: fetch_user_dict\n")
    try:
        steps, user_dict, is_encrypted = fetch_user_dict(session_id, yaml_filename, secret=secret)
        assert user_dict is not None
    except:
        return jsonify(success=False)
    if (not DEBUG) and '_internal' in user_dict and 'misc' in user_dict['_internal'] and 'variable_access' in user_dict['_internal']['misc'] and user_dict['_internal']['misc']['variable_access'] is False:
        return jsonify(success=False)
    variables = docassemble.base.functions.serializable_dict(user_dict, include_internal=True)
    #variables['_internal'] = docassemble.base.functions.serializable_dict(user_dict['_internal'])
    return jsonify(success=True, variables=variables, steps=steps, encrypted=is_encrypted, uid=session_id, i=yaml_filename)

@app.route("/", methods=['GET'])
def rootindex():
    if current_user.is_anonymous and not daconfig.get('allow anonymous access', True):
        return redirect(url_for('user.login'))
    url = daconfig.get('root redirect url', None)
    if url is not None:
        return redirect(url)
    yaml_filename = request.args.get('i', None)
    if yaml_filename is None:
        if 'default interview' not in daconfig and len(daconfig['dispatch']):
            return redirect(url_for('interview_start'))
        yaml_filename = final_default_yaml_filename
    if COOKIELESS_SESSIONS:
        return html_index()
    the_args = {}
    for key, val in request.args.items():
        the_args[key] = val
    the_args['i'] = yaml_filename
    request.args = the_args
    return index(refer=['root'])

def title_converter(content, part, status):
    if part in ('exit link', 'exit url', 'title url', 'title url opens in other window'):
        return content
    if part in ('title', 'subtitle', 'short title', 'tab title', 'exit label', 'back button label', 'corner back button label', 'logo', 'short logo', 'navigation bar html'):
        return docassemble.base.util.markdown_to_html(content, status=status, trim=True, do_terms=False)
    return docassemble.base.util.markdown_to_html(content, status=status)

@app.route("/test_embed", methods=['GET'])
@login_required
@roles_required(['admin', 'developer'])
def test_embed():
    setup_translation()
    yaml_filename = request.args.get('i', final_default_yaml_filename)
    user_dict = fresh_dictionary()
    interview = docassemble.base.interview_cache.get_interview(yaml_filename)
    interview_status = docassemble.base.parse.InterviewStatus(current_info=current_info(yaml=yaml_filename, req=request, action=None, location=None, interface='web', device_id=request.cookies.get('ds', None)))
    try:
        interview.assemble(user_dict, interview_status)
    except:
        pass
    current_language = docassemble.base.functions.get_language()
    page_title = word("Embed test")
    start_part = standard_html_start(interview_language=current_language, debug=False, bootstrap_theme=interview_status.question.interview.get_bootstrap_theme(), external=True, page_title=page_title, social=daconfig['social'], yaml_filename=yaml_filename) + global_css + additional_css(interview_status)
    scripts = standard_scripts(interview_language=current_language, external=True) + additional_scripts(interview_status, yaml_filename) + global_js
    response = make_response(render_template('pages/test_embed.html', scripts=scripts, start_part=start_part, interview_url=url_for('index', i=yaml_filename, js_target='dablock', _external=True), page_title=page_title), 200)
    response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
    return response

@app.route("/launch", methods=['GET'])
def launch():
    if COOKIELESS_SESSIONS:
        return html_index()
    code = request.args.get('c', None)
    if code is None:
        abort(403)
    the_key = 'da:resume_interview:' + str(code)
    data = r.get(the_key)
    if data is None:
        raise DAError(word("The link has expired."), code=403)
    data = json.loads(data.decode())
    if data.get('once', False):
        r.delete(the_key)
    args = {}
    for key, val in request.args.items():
        if key != 'session':
            args[key] = val
    args['i'] = data['i']
    if 'session' in data:
        update_session(data['i'], uid=data['session'])
    else:
        args['new_session'] = '1'
    request.args = args
    return index(refer=['launch'])

@app.route("/resume", methods=['POST'])
@csrf.exempt
def resume():
    post_data = request.get_json(silent=True)
    if post_data is None:
        post_data = request.form.copy()
    if 'session' not in post_data or 'i' not in post_data:
        abort(403)
    update_session(post_data['i'], uid=post_data['session'])
    del post_data['session']
    if 'ajax' in post_data:
        ajax_value = int(post_data['ajax'])
        del post_data['ajax']
        if ajax_value:
            return jsonify(action='redirect', url=url_for('index', **post_data), csrf_token=generate_csrf())
    return redirect(url_for('index', **post_data))

def tidy_action(action):
    result = {}
    if not isinstance(action, dict):
        return result
    if 'action' in action:
        result['action'] = action['action']
    if 'arguments' in action:
        result['arguments'] = action['arguments']
    return result

def make_response_wrapper(set_cookie, secret, set_device_id, device_id, expire_visitor_secret):
    def the_wrapper(response):
        if set_cookie:
            response.set_cookie('secret', secret, httponly=True, secure=app.config['SESSION_COOKIE_SECURE'], samesite=app.config['SESSION_COOKIE_SAMESITE'])
        if expire_visitor_secret:
            response.set_cookie('visitor_secret', '', expires=0)
        if set_device_id:
            response.set_cookie('ds', device_id, httponly=True, secure=app.config['SESSION_COOKIE_SECURE'], samesite=app.config['SESSION_COOKIE_SAMESITE'], expires=datetime.datetime.now() + datetime.timedelta(weeks=520))
    return the_wrapper

def populate_social(social, metadata):
    for key in ('image', 'description'):
        if key in metadata:
            if metadata[key] is None:
                if key in social:
                    del social[key]
            elif isinstance(metadata[key], str):
                social[key] = metadata[key].replace('\n', ' ').replace('"', '&quot;').strip()
    for key in ('og', 'fb', 'twitter'):
        if key in metadata and isinstance(metadata[key], dict):
            for subkey, val in metadata[key].items():
                if val is None:
                    if subkey in social[key]:
                        del social[key][subkey]
                elif isinstance(val, str):
                    social[key][subkey] = val.replace('\n', ' ').replace('"', '&quot;').strip()

if COOKIELESS_SESSIONS:
    index_path = '/i'
    html_index_path = '/interview'
else:
    index_path = '/interview'
    html_index_path = '/i'

def refresh_or_continue(interview, post_data):
    return_val = False
    try:
        if interview.questions_by_name[post_data['_question_name']].fields[0].choices[int(post_data['X211bHRpcGxlX2Nob2ljZQ'])]['key'].question_type in ('refresh', 'continue'):
            return_val = True
    except:
        pass
    return return_val

@app.route(index_path, methods=['POST', 'GET'])
def index(action_argument=None, refer=None):
    is_ajax = bool(request.method == 'POST' and 'ajax' in request.form and int(request.form['ajax']))
    docassemble.base.functions.this_thread.misc['call'] = refer
    return_fake_html = False
    if (request.method == 'POST' and 'json' in request.form and as_int(request.form['json'])) or ('json' in request.args and as_int(request.args['json'])):
        the_interface = 'json'
        is_json = True
        is_js = False
        js_target = False
    elif 'js_target' in request.args and request.args['js_target'] != '':
        the_interface = 'web'
        is_json = False
        docassemble.base.functions.this_thread.misc['jsembed'] = request.args['js_target']
        if is_ajax:
            js_target = False
        else:
            js_target = request.args['js_target']
            is_js = True
    else:
        the_interface = 'web'
        is_json = False
        is_js = False
        js_target = False
    if current_user.is_anonymous:
        if 'tempuser' not in session:
            new_temp_user = TempUser()
            db.session.add(new_temp_user)
            db.session.commit()
            session['tempuser'] = new_temp_user.id
    else:
        if 'user_id' not in session:
            session['user_id'] = current_user.id
    expire_visitor_secret = False
    if 'visitor_secret' in request.cookies:
        if 'session' in request.args:
            secret = request.cookies.get('secret', None)
            expire_visitor_secret = True
        else:
            secret = request.cookies['visitor_secret']
    else:
        secret = request.cookies.get('secret', None)
    use_cache = int(request.args.get('cache', 1))
    reset_interview = int(request.args.get('reset', 0))
    new_interview = int(request.args.get('new_session', 0))
    if secret is None:
        secret = random_string(16)
        set_cookie = True
        set_device_id = True
    else:
        secret = str(secret)
        set_cookie = False
        set_device_id = False
    device_id = request.cookies.get('ds', None)
    if device_id is None:
        device_id = random_string(16)
        set_device_id = True
    steps = 1
    need_to_reset = False
    if 'i' not in request.args and 'state' in request.args:
        try:
            yaml_filename = re.sub(r'\^.*', '', from_safeid(request.args['state']))
        except:
            yaml_filename = guess_yaml_filename()
    else:
        yaml_filename = request.args.get('i', guess_yaml_filename())
    if yaml_filename is None:
        if current_user.is_anonymous and not daconfig.get('allow anonymous access', True):
            sys.stderr.write("Redirecting to login because no YAML filename provided and no anonymous access is allowed.\n")
            return redirect(url_for('user.login'))
        if len(daconfig['dispatch']) > 0:
            sys.stderr.write("Redirecting to dispatch page because no YAML filename provided.\n")
            return redirect(url_for('interview_start'))
        yaml_filename = final_default_yaml_filename
    session_info = get_session(yaml_filename)
    session_parameter = request.args.get('session', None)
    if session_info is None or reset_interview or new_interview:
        was_new = True
        if (PREVENT_DEMO) and (yaml_filename.startswith('docassemble.base:') or yaml_filename.startswith('docassemble.demo:')) and (current_user.is_anonymous or not current_user.has_role('admin', 'developer')):
            raise DAError(word("Not authorized"), code=403)
        if current_user.is_anonymous and not daconfig.get('allow anonymous access', True):
            sys.stderr.write("Redirecting to login because no anonymous access allowed.\n")
            return redirect(url_for('user.login', next=url_for('index', **request.args)))
        if yaml_filename.startswith('docassemble.playground'):
            if not app.config['ENABLE_PLAYGROUND']:
                raise DAError(word("Not authorized"), code=403)
        else:
            yaml_filename = re.sub(r':([^\/]+)$', r':data/questions/\1', yaml_filename)
        show_flash = False
        interview = docassemble.base.interview_cache.get_interview(yaml_filename)
        if session_info is None and request.args.get('from_list', None) is None and not yaml_filename.startswith("docassemble.playground") and not yaml_filename.startswith("docassemble.base") and not yaml_filename.startswith("docassemble.demo") and SHOW_LOGIN and not new_interview and len(session['sessions']) > 0:
            show_flash = True
        if current_user.is_authenticated and current_user.has_role('admin', 'developer', 'advocate'):
            show_flash = False
        if session_parameter is None:
            if show_flash:
                if current_user.is_authenticated:
                    #word("Starting a new interview.  To go back to your previous interview, go to My Interviews on the menu.")
                    message = "Starting a new interview.  To go back to your previous interview, go to My Interviews on the menu."
                else:
                    #word("Starting a new interview.  To go back to your previous interview, log in to see a list of your interviews.")
                    message = "Starting a new interview.  To go back to your previous interview, log in to see a list of your interviews."
            if reset_interview and session_info is not None:
                reset_user_dict(session_info['uid'], yaml_filename)
            unique_sessions = interview.consolidated_metadata.get('sessions are unique', False)
            if unique_sessions is not False and not current_user.is_authenticated:
                delete_session_for_interview(yaml_filename)
                flash(word("You need to be logged in to access this interview."), "info")
                sys.stderr.write("Redirecting to login because sessions are unique.\n")
                return redirect(url_for('user.login', next=url_for('index', **request.args)))
            if interview.consolidated_metadata.get('temporary session', False):
                if session_info is not None:
                    reset_user_dict(session_info['uid'], yaml_filename)
                if current_user.is_authenticated:
                    while True:
                        session_id, encrypted = get_existing_session(yaml_filename, secret)
                        if session_id:
                            reset_user_dict(session_id, yaml_filename)
                        else:
                            break
                    reset_interview = 1
            if current_user.is_anonymous:
                if (not interview.allowed_to_initiate(is_anonymous=True)) or (not interview.allowed_to_access(is_anonymous=True)):
                    delete_session_for_interview(yaml_filename)
                    flash(word("You need to be logged in to access this interview."), "info")
                    sys.stderr.write("Redirecting to login because anonymous user not allowed to access this interview.\n")
                    return redirect(url_for('user.login', next=url_for('index', **request.args)))
            elif not interview.allowed_to_initiate(has_roles=[role.name for role in current_user.roles]):
                delete_session_for_interview(yaml_filename)
                raise DAError(word("You are not allowed to access this interview."), code=403)
            elif not interview.allowed_to_access(has_roles=[role.name for role in current_user.roles]):
                raise DAError(word('You are not allowed to access this interview.'), code=403)
            session_id = None
            if reset_interview == 2:
                delete_session_sessions()
            if (not reset_interview) and (unique_sessions is True or (isinstance(unique_sessions, list) and len(unique_sessions) and current_user.has_role(*unique_sessions))):
                session_id, encrypted = get_existing_session(yaml_filename, secret)
            if session_id is None:
                user_code, user_dict = reset_session(yaml_filename, secret)
                add_referer(user_dict)
                save_user_dict(user_code, user_dict, yaml_filename, secret=secret)
                release_lock(user_code, yaml_filename)
                need_to_reset = True
            session_info = get_session(yaml_filename)
        else:
            unique_sessions = interview.consolidated_metadata.get('sessions are unique', False)
            if unique_sessions is not False and not current_user.is_authenticated:
                delete_session_for_interview(yaml_filename)
                flash(word("You need to be logged in to access this interview."), "info")
                sys.stderr.write("Redirecting to login because sessions are unique.\n")
                return redirect(url_for('user.login', next=url_for('index', **request.args)))
            if current_user.is_anonymous:
                if (not interview.allowed_to_initiate(is_anonymous=True)) or (not interview.allowed_to_access(is_anonymous=True)):
                    delete_session_for_interview(yaml_filename)
                    flash(word("You need to be logged in to access this interview."), "info")
                    sys.stderr.write("Redirecting to login because anonymous user not allowed to access this interview.\n")
                    return redirect(url_for('user.login', next=url_for('index', **request.args)))
            elif not interview.allowed_to_initiate(has_roles=[role.name for role in current_user.roles]):
                delete_session_for_interview(yaml_filename)
                raise DAError(word("You are not allowed to access this interview."), code=403)
            elif not interview.allowed_to_access(has_roles=[role.name for role in current_user.roles]):
                raise DAError(word('You are not allowed to access this interview.'), code=403)
            if reset_interview:
                reset_user_dict(session_parameter, yaml_filename)
                if reset_interview == 2:
                    delete_session_sessions()
                user_code, user_dict = reset_session(yaml_filename, secret)
                add_referer(user_dict)
                save_user_dict(user_code, user_dict, yaml_filename, secret=secret)
                release_lock(user_code, yaml_filename)
                session_info = get_session(yaml_filename)
                need_to_reset = True
            else:
                session_info = update_session(yaml_filename, uid=session_parameter)
                need_to_reset = True
            if show_flash:
                if current_user.is_authenticated:
                    #word("Entering a different interview.  To go back to your previous interview, go to My Interviews on the menu.")
                    message = "Entering a different interview.  To go back to your previous interview, go to My Interviews on the menu."
                else:
                    #word("Entering a different interview.  To go back to your previous interview, log in to see a list of your interviews.")
                    message = "Entering a different interview.  To go back to your previous interview, log in to see a list of your interviews."
        if show_flash:
            flash(word(message), 'info')
    else:
        was_new = False
        if session_parameter is not None and not need_to_reset:
            session_info = update_session(yaml_filename, uid=session_parameter)
            need_to_reset = True
    user_code = session_info['uid']
    encrypted = session_info['encrypted']
    obtain_lock(user_code, yaml_filename)
    try:
        steps, user_dict, is_encrypted = fetch_user_dict(user_code, yaml_filename, secret=secret)
    except Exception as the_err:
        try:
            sys.stderr.write("index: there was an exception " + str(the_err.__class__.__name__) + ": " + str(the_err) + " after fetch_user_dict with %s and %s, so we need to reset\n" % (user_code, yaml_filename))
        except:
            sys.stderr.write("index: there was an exception " + str(the_err.__class__.__name__) + " after fetch_user_dict with %s and %s, so we need to reset\n" % (user_code, yaml_filename))
        release_lock(user_code, yaml_filename)
        logmessage("index: dictionary fetch failed")
        clear_session(yaml_filename)
        if session_parameter is not None:
            redirect_url = daconfig.get('session error redirect url', None)
            if isinstance(redirect_url, str) and redirect_url:
                redirect_url = redirect_url.format(i=urllibquote(yaml_filename), error=urllibquote('answers_fetch_fail'))
                sys.stderr.write("Session error because failure to get user dictionary.\n")
                return do_redirect(redirect_url, is_ajax, is_json, js_target)
        sys.stderr.write("Redirecting back to index because of failure to get user dictionary.\n")
        response = do_redirect(url_for('index', i=yaml_filename), is_ajax, is_json, js_target)
        if session_parameter is not None:
            flash(word("Unable to retrieve interview session.  Starting a new session instead."), "error")
        return response
    if user_dict is None:
        sys.stderr.write("index: no user_dict found after fetch_user_dict with %s and %s, so we need to reset\n" % (user_code, yaml_filename))
        release_lock(user_code, yaml_filename)
        logmessage("index: dictionary fetch returned no results")
        clear_session(yaml_filename)
        redirect_url = daconfig.get('session error redirect url', None)
        if isinstance(redirect_url, str) and redirect_url:
            redirect_url = redirect_url.format(i=urllibquote(yaml_filename), error=urllibquote('answers_missing'))
            sys.stderr.write("Session error because user dictionary was None.\n")
            return do_redirect(redirect_url, is_ajax, is_json, js_target)
        sys.stderr.write("Redirecting back to index because user dictionary was None.\n")
        response = do_redirect(url_for('index', i=yaml_filename), is_ajax, is_json, js_target)
        flash(word("Unable to locate interview session.  Starting a new session instead."), "error")
        return response
    if encrypted != is_encrypted:
        update_session(yaml_filename, encrypted=is_encrypted)
        encrypted = is_encrypted
    action = None
    if user_dict.get('multi_user', False) is True and encrypted is True:
        encrypted = False
        update_session(yaml_filename, encrypted=encrypted)
        decrypt_session(secret, user_code=user_code, filename=yaml_filename)
    if user_dict.get('multi_user', False) is False and encrypted is False:
        encrypt_session(secret, user_code=user_code, filename=yaml_filename)
        encrypted = True
        update_session(yaml_filename, encrypted=encrypted)
    if not session_info['key_logged']:
        save_user_dict_key(user_code, yaml_filename)
        update_session(yaml_filename, key_logged=True)
    if '_action' in request.form and 'in error' not in session:
        action = tidy_action(json.loads(myb64unquote(request.form['_action'])))
        no_defs = True
    elif 'action' in request.args and 'in error' not in session:
        action = tidy_action(json.loads(myb64unquote(request.args['action'])))
        no_defs = True
    elif action_argument:
        action = tidy_action(action_argument)
        no_defs = False
    else:
        no_defs = False
    url_args_changed = False
    if len(request.args) > 0:
        for argname in request.args:
            if argname in reserved_argnames:
                continue
            if not url_args_changed:
                old_url_args = copy.deepcopy(user_dict['url_args'])
                url_args_changed = True
            exec("url_args[" + repr(argname) + "] = " + repr(request.args.get(argname)), user_dict)
        if url_args_changed:
            if old_url_args == user_dict['url_args']:
                url_args_changed = False
    index_params = dict(i=yaml_filename)
    if analytics_configured:
        for argname in request.args:
            if argname in ('utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content'):
                index_params[argname] = request.args[argname]
    if need_to_reset or set_device_id:
        if use_cache == 0:
            docassemble.base.parse.interview_source_from_string(yaml_filename).update_index()
        response_wrapper = make_response_wrapper(set_cookie, secret, set_device_id, device_id, expire_visitor_secret)
    else:
        response_wrapper = None
    disregard_input = not bool(request.method == 'POST' and not no_defs)
    if disregard_input:
        post_data = {}
    else:
        post_data = request.form.copy()
    if current_user.is_anonymous:
        the_user_id = 't' + str(session['tempuser'])
    else:
        the_user_id = current_user.id
    if '_track_location' in post_data and post_data['_track_location']:
        the_location = json.loads(post_data['_track_location'])
    else:
        the_location = None
    interview = docassemble.base.interview_cache.get_interview(yaml_filename)
    interview_status = docassemble.base.parse.InterviewStatus(current_info=current_info(yaml=yaml_filename, req=request, action=None, location=the_location, interface=the_interface, session_info=session_info, secret=secret, device_id=device_id), tracker=user_dict['_internal']['tracker'])
    old_user_dict = None
    if '_back_one' in post_data and steps > 1:
        ok_to_go_back = True
        if STRICT_MODE:
            interview.assemble(user_dict, interview_status=interview_status)
            if not interview_status.question.can_go_back:
                ok_to_go_back = False
        if ok_to_go_back:
            old_user_dict = user_dict
            steps, user_dict, is_encrypted = fetch_previous_user_dict(user_code, yaml_filename, secret)
            if encrypted != is_encrypted:
                encrypted = is_encrypted
                update_session(yaml_filename, encrypted=encrypted)
            action = None
            interview_status = docassemble.base.parse.InterviewStatus(current_info=current_info(yaml=yaml_filename, req=request, action=action, location=the_location, interface=the_interface, session_info=session_info, secret=secret, device_id=device_id), tracker=user_dict['_internal']['tracker'])
            post_data = {}
            disregard_input = True
    known_varnames = {}
    if '_varnames' in post_data:
        known_varnames = json.loads(myb64unquote(post_data['_varnames']))
    if '_visible' in post_data and post_data['_visible'] != "":
        visible_field_names = json.loads(myb64unquote(post_data['_visible']))
    else:
        visible_field_names = []
    known_varnames_visible = {}
    for key, val in known_varnames.items():
        if key in visible_field_names:
            known_varnames_visible[key] = val
    all_field_numbers = {}
    field_numbers = {}
    numbered_fields = {}
    visible_fields = set()
    raw_visible_fields = set()
    for field_name in visible_field_names:
        try:
            m = re.search(r'(.*)(\[[^\]]+\])$', from_safeid(field_name))
            if m:
                if safeid(m.group(1)) in known_varnames:
                    visible_fields.add(safeid(from_safeid(known_varnames[safeid(m.group(1))]) + m.group(2)))
        except Exception as the_err:
            pass
        raw_visible_fields.add(field_name)
        if field_name in known_varnames:
            visible_fields.add(known_varnames[field_name])
        else:
            visible_fields.add(field_name)
    for kv_key, kv_var in known_varnames.items():
        try:
            field_identifier = myb64unquote(kv_key)
            m = re.search(r'_field(?:_[0-9]+)?_([0-9]+)', field_identifier)
            if m:
                numbered_fields[kv_var] = kv_key
                if kv_key in raw_visible_fields or kv_var in raw_visible_fields:
                    field_numbers[kv_var] = int(m.group(1))
            m = re.search(r'_field_((?:[0-9]+_)?[0-9]+)', field_identifier)
            if m:
                if kv_var not in all_field_numbers:
                    all_field_numbers[kv_var] = set()
                if '_' in m.group(1):
                    all_field_numbers[kv_var].add(m.group(1))
                else:
                    all_field_numbers[kv_var].add(int(m.group(1)))
        except:
            logmessage("index: error where kv_key is " + str(kv_key) + " and kv_var is " + str(kv_var))
    list_collect_list = None
    if not STRICT_MODE:
        if '_list_collect_list' in post_data:
            the_list = json.loads(myb64unquote(post_data['_list_collect_list']))
            if not illegal_variable_name(the_list):
                list_collect_list = the_list
                exec(list_collect_list + '._allow_appending()', user_dict)
        if '_checkboxes' in post_data:
            checkbox_fields = json.loads(myb64unquote(post_data['_checkboxes'])) #post_data['_checkboxes'].split(",")
            for checkbox_field, checkbox_value in checkbox_fields.items():
                if checkbox_field in visible_fields and checkbox_field not in post_data and not (checkbox_field in numbered_fields and numbered_fields[checkbox_field] in post_data):
                    post_data.add(checkbox_field, checkbox_value)
        if '_empties' in post_data:
            empty_fields = json.loads(myb64unquote(post_data['_empties']))
            for empty_field in empty_fields:
                if empty_field not in post_data:
                    post_data.add(empty_field, 'None')
        else:
            empty_fields = {}
        if '_ml_info' in post_data:
            ml_info = json.loads(myb64unquote(post_data['_ml_info']))
        else:
            ml_info = {}
    something_changed = False
    if '_tracker' in post_data and re.search(r'^-?[0-9]+$', post_data['_tracker']) and user_dict['_internal']['tracker'] != int(post_data['_tracker']):
        if user_dict['_internal']['tracker'] > int(post_data['_tracker']):
            logmessage("index: the assemble function has been run since the question was posed.")
        else:
            logmessage("index: the tracker in the dictionary is behind the tracker in the question.")
        something_changed = True
        user_dict['_internal']['tracker'] = max(int(post_data['_tracker']), user_dict['_internal']['tracker'])
        interview_status.tracker = user_dict['_internal']['tracker']
    should_assemble = False
    known_datatypes = {}
    if not STRICT_MODE:
        if '_datatypes' in post_data:
            known_datatypes = json.loads(myb64unquote(post_data['_datatypes']))
            for data_type in known_datatypes.values():
                if data_type.startswith('object'):
                    should_assemble = True
    if not should_assemble:
        for key in post_data:
            if key.startswith('_') or key in ('csrf_token', 'ajax', 'json', 'informed'):
                continue
            try:
                the_key = from_safeid(key)
                if the_key.startswith('_field_'):
                    if key in known_varnames:
                        if not (known_varnames[key] in post_data and post_data[known_varnames[key]] != '' and post_data[key] == ''):
                            the_key = from_safeid(known_varnames[key])
                    else:
                        m = re.search(r'^(_field(?:_[0-9]+)?_[0-9]+)(\[.*\])', key)
                        if m:
                            base_orig_key = safeid(m.group(1))
                            if base_orig_key in known_varnames:
                                the_key = myb64unquote(known_varnames[base_orig_key]) + m.group(2)
                if key_requires_preassembly.search(the_key):
                    if the_key == '_multiple_choice' and '_question_name' in post_data:
                        if refresh_or_continue(interview, post_data):
                            continue
                    should_assemble = True
                    break
            except Exception as the_err:
                logmessage("index: bad key was " + str(key) + " and error was " + the_err.__class__.__name__)
                try:
                    logmessage("index: bad key error message was " + str(the_err))
                except:
                    pass
    if not interview.from_cache and len(interview.mlfields):
        ensure_training_loaded(interview)
    debug_mode = interview.debug
    vars_set = set()
    old_values = {}
    new_values = {}
    if ('_email_attachments' in post_data and '_attachment_email_address' in post_data) or '_download_attachments' in post_data:
        should_assemble = True
    error_messages = []
    already_assembled = False
    if (STRICT_MODE and not disregard_input) or should_assemble or something_changed:
        interview.assemble(user_dict, interview_status=interview_status)
        already_assembled = True
        if STRICT_MODE and ('_question_name' not in post_data or post_data['_question_name'] != interview_status.question.name):
            if refresh_or_continue(interview, post_data) is False and action is None and len([key for key in post_data if not (key.startswith('_') or key in ('csrf_token', 'ajax', 'json', 'informed'))]) > 0:
                error_messages.append(("success", word("Input not processed.  Please try again.")))
            post_data = {}
            disregard_input = True
        elif should_assemble and '_question_name' in post_data and post_data['_question_name'] != interview_status.question.name:
            logmessage("index: not the same question name: " + str(post_data['_question_name']) + " versus " + str(interview_status.question.name))
            if REQUIRE_IDEMPOTENT:
                error_messages.append(("success", word("Input not processed because the question changed.  Please continue.")))
                post_data = {}
                disregard_input = True
    if STRICT_MODE and not disregard_input:
        field_info = interview_status.get_field_info()
        known_datatypes = field_info['datatypes']
        list_collect_list = field_info['list_collect_list']
        if list_collect_list is not None:
            exec(list_collect_list + '._allow_appending()' , user_dict)
        for checkbox_field, checkbox_value in field_info['checkboxes'].items():
            if checkbox_field in visible_fields and checkbox_field not in post_data and not (checkbox_field in numbered_fields and numbered_fields[checkbox_field] in post_data):
                post_data.add(checkbox_field, checkbox_value)
        empty_fields = field_info['hiddens']
        for empty_field in empty_fields:
            if empty_field not in post_data:
                post_data.add(empty_field, 'None')
        ml_info = field_info['ml_info']
        authorized_fields = [from_safeid(field.saveas) for field in interview_status.get_fields_and_sub_fields_and_collect_fields(user_dict) if hasattr(field, 'saveas')]
        if 'allowed_to_set' in interview_status.extras:
            authorized_fields.extend(interview_status.extras['allowed_to_set'])
        if interview_status.question.question_type == "multiple_choice":
            authorized_fields.append('_multiple_choice')
        authorized_fields = set(authorized_fields).union(interview_status.get_all_fields_used(user_dict))
        if interview_status.extras.get('list_collect_is_final', False) and interview_status.extras['list_collect'].auto_gather:
            if interview_status.extras['list_collect'].ask_number:
                authorized_fields.add(interview_status.extras['list_collect'].instanceName + ".target_number")
            else:
                authorized_fields.add(interview_status.extras['list_collect'].instanceName + ".there_is_another")
    else:
        if STRICT_MODE:
            empty_fields = []
        authorized_fields = set()
    changed = False
    if '_null_question' in post_data:
        changed = True
    if '_email_attachments' in post_data and '_attachment_email_address' in post_data:
        success = False
        attachment_email_address = post_data['_attachment_email_address'].strip()
        if '_attachment_include_editable' in post_data:
            include_editable = bool(post_data['_attachment_include_editable'] == 'True')
            del post_data['_attachment_include_editable']
        else:
            include_editable = False
        del post_data['_email_attachments']
        del post_data['_attachment_email_address']
        if len(interview_status.attachments) > 0:
            attached_file_count = 0
            attachment_info = []
            for the_attachment in interview_status.attachments:
                file_formats = []
                if 'pdf' in the_attachment['valid_formats'] or '*' in the_attachment['valid_formats']:
                    file_formats.append('pdf')
                if include_editable or 'pdf' not in file_formats:
                    if 'rtf' in the_attachment['valid_formats'] or '*' in the_attachment['valid_formats']:
                        file_formats.append('rtf')
                    if 'docx' in the_attachment['valid_formats']:
                        file_formats.append('docx')
                    if 'rtf to docx' in the_attachment['valid_formats']:
                        file_formats.append('rtf to docx')
                    if 'md' in the_attachment['valid_formats']:
                        file_formats.append('md')
                if 'raw' in the_attachment['valid_formats']:
                    file_formats.append('raw')
                for the_format in file_formats:
                    if the_format == 'raw':
                        attachment_info.append({'filename': str(the_attachment['filename']) + '.' + the_attachment['raw'], 'number': the_attachment['file'][the_format], 'mimetype': the_attachment['mimetype'][the_format], 'attachment': the_attachment})
                    else:
                        attachment_info.append({'filename': str(the_attachment['filename']) + '.' + str(docassemble.base.parse.extension_of_doc_format[the_format]), 'number': the_attachment['file'][the_format], 'mimetype': the_attachment['mimetype'][the_format], 'attachment': the_attachment})
                    attached_file_count += 1
            worker_key = 'da:worker:uid:' + str(user_code) + ':i:' + str(yaml_filename) + ':userid:' + str(the_user_id)
            for email_address in re.split(r' *[,;] *', attachment_email_address):
                try:
                    result = docassemble.webapp.worker.email_attachments.delay(user_code, email_address, attachment_info, docassemble.base.functions.get_language(), subject=interview_status.extras.get('email_subject', None), body=interview_status.extras.get('email_body', None), html=interview_status.extras.get('email_html', None))
                    r.rpush(worker_key, result.id)
                    success = True
                except Exception as errmess:
                    success = False
                    logmessage("index: failed with " + str(errmess))
                    break
            if success:
                flash(word("Your documents will be e-mailed to") + " " + str(attachment_email_address) + ".", 'success')
            else:
                flash(word("Unable to e-mail your documents to") + " " + str(attachment_email_address) + ".", 'error')
        else:
            flash(word("Unable to find documents to e-mail."), 'error')
    if '_download_attachments' in post_data:
        success = False
        if '_attachment_include_editable' in post_data:
            include_editable = bool(post_data['_attachment_include_editable'] == 'True')
            del post_data['_attachment_include_editable']
        else:
            include_editable = False
        del post_data['_download_attachments']
        if len(interview_status.attachments) > 0:
            attached_file_count = 0
            files_to_zip = []
            if 'zip_filename' in interview_status.extras and interview_status.extras['zip_filename']:
                zip_file_name = interview_status.extras['zip_filename']
            else:
                zip_file_name = 'file.zip'
            for the_attachment in interview_status.attachments:
                file_formats = []
                if 'pdf' in the_attachment['valid_formats'] or '*' in the_attachment['valid_formats']:
                    file_formats.append('pdf')
                if include_editable or 'pdf' not in file_formats:
                    if 'rtf' in the_attachment['valid_formats'] or '*' in the_attachment['valid_formats']:
                        file_formats.append('rtf')
                    if 'docx' in the_attachment['valid_formats']:
                        file_formats.append('docx')
                    if 'rtf to docx' in the_attachment['valid_formats']:
                        file_formats.append('rtf to docx')
                for the_format in file_formats:
                    files_to_zip.append(str(the_attachment['file'][the_format]))
                    attached_file_count += 1
            the_zip_file = docassemble.base.util.zip_file(*files_to_zip, filename=zip_file_name)
            response = send_file(the_zip_file.path(), mimetype='application/zip', as_attachment=True, attachment_filename=zip_file_name)
            response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
            if response_wrapper:
                response_wrapper(response)
            return response
    if '_the_image' in post_data and (STRICT_MODE is False or interview_status.question.question_type == 'signature'):
        if STRICT_MODE:
            file_field = from_safeid(field_info['signature_saveas'])
        else:
            file_field = from_safeid(post_data['_save_as'])
        if illegal_variable_name(file_field):
            error_messages.append(("error", "Error: Invalid character in file_field: " + str(file_field)))
        else:
            if not already_assembled:
                interview.assemble(user_dict, interview_status)
                already_assembled = True
            initial_string = 'import docassemble.base.util'
            try:
                exec(initial_string, user_dict)
            except Exception as errMess:
                error_messages.append(("error", "Error: " + str(errMess)))
            file_field_tr = sub_indices(file_field, user_dict)
            if '_success' in post_data and post_data['_success']:
                theImage = base64.b64decode(re.search(r'base64,(.*)', post_data['_the_image']).group(1) + '==')
                filename = secure_filename('canvas.png')
                file_number = get_new_file_number(user_code, filename, yaml_file_name=yaml_filename)
                extension, mimetype = get_ext_and_mimetype(filename)
                new_file = SavedFile(file_number, extension=extension, fix=True, should_not_exist=True)
                new_file.write_content(theImage, binary=True)
                new_file.finalize()
                the_string = file_field + " = docassemble.base.util.DAFile(" + repr(file_field_tr) + ", filename='" + str(filename) + "', number=" + str(file_number) + ", mimetype='" + str(mimetype) + "', make_pngs=True, extension='" + str(extension) + "')"
            else:
                the_string = file_field + " = docassemble.base.util.DAFile(" + repr(file_field_tr) + ")"
            process_set_variable(file_field, user_dict, vars_set, old_values)
            try:
                exec(the_string, user_dict)
                changed = True
            except Exception as errMess:
                try:
                    sys.stderr.write(errMess.__class__.__name__ + ": " + str(errMess) + " after running " + the_string)
                except:
                    pass
                error_messages.append(("error", "Error: " + errMess.__class__.__name__ + ": " + str(errMess)))
    if '_next_action_to_set' in post_data:
        next_action_to_set = json.loads(myb64unquote(post_data['_next_action_to_set']))
    else:
        next_action_to_set = None
    next_action = None
    if '_question_name' in post_data and post_data['_question_name'] in interview.questions_by_name:
        if already_assembled:
            the_question = interview_status.question
        else:
            the_question = interview.questions_by_name[post_data['_question_name']]
        if not already_assembled:
            uses_permissions = False
            for the_field in the_question.fields:
                if hasattr(the_field, 'permissions'):
                    uses_permissions = True
            if uses_permissions or the_question.validation_code is not None:
                interview.assemble(user_dict, interview_status)
            else:
                for the_field in the_question.fields:
                    if hasattr(the_field, 'validate'):
                        interview.assemble(user_dict, interview_status)
                        break
    elif already_assembled:
        the_question = interview_status.question
    else:
        the_question = None
    key_to_orig_key = {}
    for orig_key in copy.deepcopy(post_data):
        if orig_key in ('_checkboxes', '_empties', '_ml_info', '_back_one', '_files', '_files_inline', '_question_name', '_the_image', '_save_as', '_success', '_datatypes', '_event', '_visible', '_tracker', '_track_location', '_varnames', '_next_action', '_next_action_to_set', 'ajax', 'json', 'informed', 'csrf_token', '_action', '_order_changes', '_collect', '_collect_delete', '_list_collect_list', '_null_question') or orig_key.startswith('_ignore'):
            continue
        try:
            key = myb64unquote(orig_key)
        except:
            continue
        if key.startswith('_field_'):
            if orig_key in known_varnames:
                if not (known_varnames[orig_key] in post_data and post_data[known_varnames[orig_key]] != '' and post_data[orig_key] == ''):
                    post_data[known_varnames[orig_key]] = post_data[orig_key]
                    key_to_orig_key[from_safeid(known_varnames[orig_key])] = orig_key
            else:
                m = re.search(r'^(_field(?:_[0-9]+)?_[0-9]+)(\[.*\])', key)
                if m:
                    base_orig_key = safeid(m.group(1))
                    if base_orig_key in known_varnames:
                        the_key = myb64unquote(known_varnames[base_orig_key]) + m.group(2)
                        key_to_orig_key[the_key] = orig_key
                        full_key = safeid(the_key)
                        post_data[full_key] = post_data[orig_key]
        if key.endswith('.gathered'):
            if STRICT_MODE and key not in authorized_fields:
                raise DAError("The variable " + repr(key) + " was not in the allowed fields, which were " + repr(authorized_fields))
            objname = re.sub(r'\.gathered$', '', key)
            if illegal_variable_name(objname):
                error_messages.append(("error", "Error: Invalid key " + objname))
                break
            try:
                eval(objname, user_dict)
            except:
                safe_objname = safeid(objname)
                if safe_objname in known_datatypes:
                    if known_datatypes[safe_objname] in ('object_multiselect', 'object_checkboxes'):
                        docassemble.base.parse.ensure_object_exists(objname, 'object_checkboxes', user_dict)
                    elif known_datatypes[safe_objname] in ('multiselect', 'checkboxes'):
                        docassemble.base.parse.ensure_object_exists(objname, known_datatypes[safe_objname], user_dict)
    field_error = {}
    validated = True
    pre_user_dict = user_dict
    imported_core = False
    special_question = None
    for orig_key in post_data:
        if orig_key in ('_checkboxes', '_empties', '_ml_info', '_back_one', '_files', '_files_inline', '_question_name', '_the_image', '_save_as', '_success', '_datatypes', '_event', '_visible', '_tracker', '_track_location', '_varnames', '_next_action', '_next_action_to_set', 'ajax', 'json', 'informed', 'csrf_token', '_action', '_order_changes', '', '_collect', '_collect_delete', '_list_collect_list', '_null_question') or orig_key.startswith('_ignore'):
            continue
        data = post_data[orig_key]
        try:
            key = myb64unquote(orig_key)
        except:
            raise DAError("index: invalid name " + str(orig_key))
        if key.startswith('_field_'):
            continue
        bracket_expression = None
        if orig_key in empty_fields:
            set_to_empty = empty_fields[orig_key]
        else:
            set_to_empty = None
        if match_brackets.search(key):
            match = match_inside_and_outside_brackets.search(key)
            try:
                key = match.group(1)
            except:
                try:
                    error_message = "index: invalid bracket name " + str(match.group(1)) + " in " + repr(key)
                except:
                    error_message = "index: invalid bracket name in " + repr(key)
                raise DAError(error_message)
            real_key = safeid(key)
            b_match = match_inside_brackets.search(match.group(2))
            if b_match:
                if b_match.group(1) in ('B', 'R'):
                    try:
                        bracket_expression = from_safeid(b_match.group(2))
                    except:
                        bracket_expression = b_match.group(2)
                else:
                    bracket_expression = b_match.group(2)
            bracket = match_inside_brackets.sub(process_bracket_expression, match.group(2))
            parse_result = docassemble.base.parse.parse_var_name(key)
            if not parse_result['valid']:
                error_messages.append(("error", "Error: Invalid key " + key + ": " + parse_result['reason']))
                break
            pre_bracket_key = key
            key = key + bracket
            core_key_name = parse_result['final_parts'][0]
            whole_key = core_key_name + parse_result['final_parts'][1]
            real_key = safeid(whole_key)
            if STRICT_MODE and (pre_bracket_key not in authorized_fields or pre_bracket_key + '.gathered' not in authorized_fields) and (key not in authorized_fields):
                raise DAError("The variables " + repr(pre_bracket_key) + " and " + repr(key) + " were not in the allowed fields, which were " + repr(authorized_fields))
            if illegal_variable_name(whole_key) or illegal_variable_name(core_key_name) or illegal_variable_name(key):
                error_messages.append(("error", "Error: Invalid key " + whole_key))
                break
            if whole_key in user_dict:
                it_exists = True
            else:
                try:
                    the_object = eval(whole_key, user_dict)
                    it_exists = True
                except:
                    it_exists = False
            if not it_exists:
                method = None
                commands = []
                if parse_result['final_parts'][1] != '':
                    if parse_result['final_parts'][1][0] == '.':
                        try:
                            core_key = eval(core_key_name, user_dict)
                            if hasattr(core_key, 'instanceName'):
                                method = 'attribute'
                        except:
                            pass
                    elif parse_result['final_parts'][1][0] == '[':
                        try:
                            core_key = eval(core_key_name, user_dict)
                            if hasattr(core_key, 'instanceName'):
                                method = 'index'
                        except:
                            pass
                datatype = known_datatypes.get(real_key, None)
                if not imported_core:
                    commands.append("import docassemble.base.util")
                    imported_core = True
                if method == 'attribute':
                    attribute_name = parse_result['final_parts'][1][1:]
                    if datatype in ('multiselect', 'checkboxes'):
                        commands.append(core_key_name + ".initializeAttribute(" + repr(attribute_name) + ", docassemble.base.util.DADict, auto_gather=False, gathered=True)")
                    elif datatype in ('object_multiselect', 'object_checkboxes'):
                        commands.append(core_key_name + ".initializeAttribute(" + repr(attribute_name) + ", docassemble.base.util.DAList, auto_gather=False, gathered=True)")
                    process_set_variable(core_key_name + '.' + attribute_name, user_dict, vars_set, old_values)
                elif method == 'index':
                    index_name = parse_result['final_parts'][1][1:-1]
                    orig_index_name = index_name
                    if index_name in ('i', 'j', 'k', 'l', 'm', 'n'):
                        index_name = repr(user_dict.get(index_name, index_name))
                    if datatype in ('multiselect', 'checkboxes'):
                        commands.append(core_key_name + ".initializeObject(" + index_name + ", docassemble.base.util.DADict, auto_gather=False, gathered=True)")
                    elif datatype in ('object_multiselect', 'object_checkboxes'):
                        commands.append(core_key_name + ".initializeObject(" + index_name + ", docassemble.base.util.DAList, auto_gather=False, gathered=True)")
                    process_set_variable(core_key_name + '[' + orig_index_name + ']', user_dict, vars_set, old_values)
                else:
                    whole_key_tr = sub_indices(whole_key, user_dict)
                    if datatype in ('multiselect', 'checkboxes'):
                        commands.append(whole_key + ' = docassemble.base.util.DADict(' + repr(whole_key_tr) + ', auto_gather=False, gathered=True)')
                    elif datatype in ('object_multiselect', 'object_checkboxes'):
                        commands.append(whole_key + ' = docassemble.base.util.DAList(' + repr(whole_key_tr) + ', auto_gather=False, gathered=True)')
                    process_set_variable(whole_key, user_dict, vars_set, old_values)
                for command in commands:
                    exec(command, user_dict)
        else:
            real_key = orig_key
            parse_result = docassemble.base.parse.parse_var_name(key)
            if not parse_result['valid']:
                error_messages.append(("error", "Error: Invalid character in key: " + key))
                break
            if STRICT_MODE and key not in authorized_fields:
                raise DAError("The variable " + repr(key) + " was not in the allowed fields, which were " + repr(authorized_fields))
        if illegal_variable_name(key):
            error_messages.append(("error", "Error: Invalid key " + key))
            break
        do_append = False
        do_opposite = False
        is_ml = False
        is_date = False
        is_object = False
        test_data = data
        if real_key in known_datatypes:
            if known_datatypes[real_key] in ('boolean', 'multiselect', 'checkboxes'):
                if data == "True":
                    data = "True"
                    test_data = True
                else:
                    data = "False"
                    test_data = False
            elif known_datatypes[real_key] == 'threestate':
                if data == "True":
                    data = "True"
                    test_data = True
                elif data == "None":
                    data = "None"
                    test_data = None
                else:
                    data = "False"
                    test_data = False
            elif known_datatypes[real_key] in ('date', 'datetime', 'datetime-local'):
                if isinstance(data, str):
                    data = data.strip()
                    if data != '':
                        try:
                            dateutil.parser.parse(data)
                        except:
                            validated = False
                            if known_datatypes[real_key] == 'date':
                                field_error[orig_key] = word("You need to enter a valid date.")
                            else:
                                field_error[orig_key] = word("You need to enter a valid date and time.")
                            continue
                        test_data = data
                        is_date = True
                        data = 'docassemble.base.util.as_datetime(' + repr(data) + ')'
                    else:
                        data = repr('')
                else:
                    data = repr('')
            elif known_datatypes[real_key] == 'time':
                if isinstance(data, str):
                    data = data.strip()
                    if data != '':
                        try:
                            dateutil.parser.parse(data)
                        except:
                            validated = False
                            field_error[orig_key] = word("You need to enter a valid time.")
                            continue
                        test_data = data
                        is_date = True
                        data = 'docassemble.base.util.as_datetime(' + repr(data) + ').time()'
                    else:
                        data = repr('')
                else:
                    data = repr('')
            elif known_datatypes[real_key] == 'integer':
                if data.strip() == '':
                    data = 0
                try:
                    test_data = int(data)
                except:
                    validated = False
                    field_error[orig_key] = word("You need to enter a valid number.")
                    continue
                data = "int(" + repr(data) + ")"
            elif known_datatypes[real_key] in ('ml', 'mlarea'):
                is_ml = True
            elif known_datatypes[real_key] in ('number', 'float', 'currency', 'range'):
                if data == '':
                    data = 0.0
                if isinstance(data, str):
                    data = re.sub(r'[,\%]', '', data)
                try:
                    test_data = float(data)
                except:
                    validated = False
                    field_error[orig_key] = word("You need to enter a valid number.")
                    continue
                data = "float(" + repr(data) + ")"
            elif known_datatypes[real_key] in ('object', 'object_radio'):
                if data == '' or set_to_empty:
                    continue
                data = "_internal['objselections'][" + repr(key) + "][" + repr(data) + "]"
            elif known_datatypes[real_key] in ('object_multiselect', 'object_checkboxes') and bracket_expression is not None:
                if data not in ('True', 'False', 'None') or set_to_empty:
                    continue
                do_append = True
                if data == 'False':
                    do_opposite = True
                data = "_internal['objselections'][" + repr(from_safeid(real_key)) + "][" + repr(bracket_expression) + "]"
            elif set_to_empty in ('object_multiselect', 'object_checkboxes'):
                continue
            elif known_datatypes[real_key] in ('file', 'files', 'camera', 'user', 'environment'):
                continue
            elif known_datatypes[real_key] in docassemble.base.functions.custom_types:
                info = docassemble.base.functions.custom_types[known_datatypes[real_key]]
                if info['is_object']:
                    is_object = True
                if set_to_empty:
                    if info['skip_if_empty']:
                        continue
                    else:
                        test_data = info['class'].empty()
                        if is_object:
                            user_dict['__DANEWOBJECT'] = data
                            data = '__DANEWOBJECT'
                        else:
                            data = repr(test_data)
                else:
                    try:
                        if not info['class'].validate(data):
                            raise DAValidationError(word("You need to enter a valid value."))
                    except DAValidationError as err:
                        validated = False
                        field_error[orig_key] = word(err)
                        continue
                    test_data = info['class'].transform(data)
                    if is_object:
                        user_dict['__DANEWOBJECT'] = test_data
                        data = '__DANEWOBJECT'
                    else:
                        data = repr(test_data)
            elif known_datatypes[real_key] == 'raw':
                if data == "None" and set_to_empty is not None:
                    test_data = None
                    data = "None"
                else:
                    test_data = data
                    data = repr(data)
            else:
                if isinstance(data, str):
                    data = BeautifulSoup(data, "html.parser").get_text('\n')
                if data == "None" and set_to_empty is not None:
                    test_data = None
                    data = "None"
                else:
                    test_data = data
                    data = repr(data)
            if known_datatypes[real_key] in ('object_multiselect', 'object_checkboxes'):
                do_append = True
        elif orig_key in known_datatypes:
            if known_datatypes[orig_key] in ('boolean', 'multiselect', 'checkboxes'):
                if data == "True":
                    data = "True"
                    test_data = True
                else:
                    data = "False"
                    test_data = False
            elif known_datatypes[orig_key] == 'threestate':
                if data == "True":
                    data = "True"
                    test_data = True
                elif data == "None":
                    data = "None"
                    test_data = None
                else:
                    data = "False"
                    test_data = False
            elif known_datatypes[orig_key] in ('date', 'datetime'):
                if isinstance(data, str):
                    data = data.strip()
                    if data != '':
                        try:
                            dateutil.parser.parse(data)
                        except:
                            validated = False
                            if known_datatypes[orig_key] == 'date':
                                field_error[orig_key] = word("You need to enter a valid date.")
                            else:
                                field_error[orig_key] = word("You need to enter a valid date and time.")
                            continue
                        test_data = data
                        is_date = True
                        data = 'docassemble.base.util.as_datetime(' + repr(data) + ')'
                    else:
                        data = repr('')
                else:
                    data = repr('')
            elif known_datatypes[orig_key] == 'time':
                if isinstance(data, str):
                    data = data.strip()
                    if data != '':
                        try:
                            dateutil.parser.parse(data)
                        except:
                            validated = False
                            field_error[orig_key] = word("You need to enter a valid time.")
                            continue
                        test_data = data
                        is_date = True
                        data = 'docassemble.base.util.as_datetime(' + repr(data) + ').time()'
                    else:
                        data = repr('')
                else:
                    data = repr('')
            elif known_datatypes[orig_key] == 'integer':
                if data == '':
                    data = 0
                test_data = int(data)
                data = "int(" + repr(data) + ")"
            elif known_datatypes[orig_key] in ('ml', 'mlarea'):
                is_ml = True
            elif known_datatypes[orig_key] in ('number', 'float', 'currency', 'range'):
                if data == '':
                    data = 0.0
                if isinstance(data, str):
                    data = re.sub(r'[,\%]', '', data)
                test_data = float(data)
                data = "float(" + repr(data) + ")"
            elif known_datatypes[orig_key] in ('object', 'object_radio'):
                if data == '' or set_to_empty:
                    continue
                data = "_internal['objselections'][" + repr(key) + "][" + repr(data) + "]"
            elif set_to_empty in ('object_multiselect', 'object_checkboxes'):
                continue
            elif real_key in known_datatypes and known_datatypes[real_key] in ('file', 'files', 'camera', 'user', 'environment'):
                continue
            elif known_datatypes[orig_key] in docassemble.base.functions.custom_types:
                info = docassemble.base.functions.custom_types[known_datatypes[orig_key]]
                if set_to_empty:
                    if info['skip_if_empty']:
                        continue
                    else:
                        test_data = info['class'].empty()
                        data = repr(test_data)
                else:
                    try:
                        if not info['class'].validate(data):
                            raise DAValidationError(word("You need to enter a valid value."))
                    except DAValidationError as err:
                        validated = False
                        field_error[orig_key] = word(str(err))
                        continue
                    test_data = info['class'].transform(data)
                    data = repr(test_data)
            else:
                if isinstance(data, str):
                    data = data.strip()
                test_data = data
                data = repr(data)
        elif key == "_multiple_choice":
            data = "int(" + repr(data) + ")"
        else:
            data = repr(data)
        if key == "_multiple_choice":
            if '_question_name' in post_data:
                question_name = post_data['_question_name']
                if question_name == 'Question_Temp':
                    key = '_internal["answers"][' + repr(interview_status.question.extended_question_name(user_dict)) + ']'
                else:
                    key = '_internal["answers"][' + repr(interview.questions_by_name[question_name].extended_question_name(user_dict)) + ']'
                    if is_integer.match(str(post_data[orig_key])):
                        the_choice = int(str(post_data[orig_key]))
                        if len(interview.questions_by_name[question_name].fields[0].choices) > the_choice and 'key' in interview.questions_by_name[question_name].fields[0].choices[the_choice] and hasattr(interview.questions_by_name[question_name].fields[0].choices[the_choice]['key'], 'question_type') and interview.questions_by_name[question_name].fields[0].choices[the_choice]['key'].question_type in ('restart', 'exit', 'logout', 'exit_logout', 'leave'):
                            special_question = interview.questions_by_name[question_name].fields[0].choices[the_choice]['key']
        if is_date:
            try:
                exec("import docassemble.base.util", user_dict)
            except Exception as errMess:
                error_messages.append(("error", "Error: " + str(errMess)))
        key_tr = sub_indices(key, user_dict)
        if is_ml:
            try:
                exec("import docassemble.base.util", user_dict)
            except Exception as errMess:
                error_messages.append(("error", "Error: " + str(errMess)))
            if orig_key in ml_info and 'train' in ml_info[orig_key]:
                if not ml_info[orig_key]['train']:
                    use_for_training = 'False'
                else:
                    use_for_training = 'True'
            else:
                use_for_training = 'True'
            if orig_key in ml_info and 'group_id' in ml_info[orig_key]:
                data = 'docassemble.base.util.DAModel(' + repr(key_tr) + ', group_id=' + repr(ml_info[orig_key]['group_id']) + ', text=' + repr(data) + ', store=' + repr(interview.get_ml_store()) + ', use_for_training=' + use_for_training + ')'
            else:
                data = 'docassemble.base.util.DAModel(' + repr(key_tr) + ', text=' + repr(data) + ', store=' + repr(interview.get_ml_store()) + ', use_for_training=' + use_for_training + ')'
        if set_to_empty:
            if set_to_empty in ('multiselect', 'checkboxes'):
                try:
                    exec("import docassemble.base.util", user_dict)
                except Exception as errMess:
                    error_messages.append(("error", "Error: " + str(errMess)))
                data = 'docassemble.base.util.DADict(' + repr(key_tr) + ', auto_gather=False, gathered=True)'
            else:
                data = 'None'
        if do_append and not set_to_empty:
            key_to_use = from_safeid(real_key)
            if illegal_variable_name(data):
                logmessage("Received illegal variable name " + str(data))
                continue
            if illegal_variable_name(key_to_use):
                logmessage("Received illegal variable name " + str(key_to_use))
                continue
            if do_opposite:
                the_string = 'if ' + data + ' in ' + key_to_use + '.elements:\n    ' + key_to_use + '.remove(' + data + ')'
            else:
                the_string = 'if ' + data + ' not in ' + key_to_use + '.elements:\n    ' + key_to_use + '.append(' + data + ')'
                if key_to_use not in new_values:
                    new_values[key_to_use] = []
                new_values[key_to_use].append(data)
        else:
            process_set_variable(key, user_dict, vars_set, old_values)
            the_string = key + ' = ' + data
            new_values[key] = data
            if orig_key in field_numbers and the_question is not None and len(the_question.fields) > field_numbers[orig_key] and hasattr(the_question.fields[field_numbers[orig_key]], 'validate'):
                field_name = safeid('_field_' + str(field_numbers[orig_key]))
                if field_name in post_data:
                    the_key = field_name
                else:
                    the_key = orig_key
                the_func = eval(the_question.fields[field_numbers[orig_key]].validate['compute'], user_dict)
                try:
                    the_result = the_func(test_data)
                    if not the_result:
                        field_error[the_key] = word("Please enter a valid value.")
                        validated = False
                        continue
                except Exception as errstr:
                    field_error[the_key] = str(errstr)
                    validated = False
                    continue
        try:
            exec(the_string, user_dict)
            changed = True
        except Exception as errMess:
            error_messages.append(("error", "Error: " + errMess.__class__.__name__ + ": " + str(errMess)))
            try:
                logmessage("Tried to run " + the_string + " and got error " + errMess.__class__.__name__ + ": " + str(errMess))
            except:
                pass
        if is_object:
            if '__DANEWOBJECT' in user_dict:
                del user_dict['__DANEWOBJECT']
        if key not in key_to_orig_key:
            key_to_orig_key[key] = orig_key
    if validated and special_question is None and not disregard_input:
        for orig_key in empty_fields:
            key = myb64unquote(orig_key)
            if STRICT_MODE and key not in authorized_fields:
                raise DAError("The variable " + repr(key) + " was not in the allowed fields, which were " + repr(authorized_fields))
            process_set_variable(key + '.gathered', user_dict, vars_set, old_values)
            if illegal_variable_name(key):
                logmessage("Received illegal variable name " + str(key))
                continue
            if empty_fields[orig_key] in ('object_multiselect', 'object_checkboxes'):
                docassemble.base.parse.ensure_object_exists(key, empty_fields[orig_key], user_dict)
                exec(key + '.clear()' , user_dict)
                exec(key + '.gathered = True' , user_dict)
            elif empty_fields[orig_key] in ('object', 'object_radio'):
                process_set_variable(key, user_dict, vars_set, old_values)
                try:
                    eval(key, user_dict)
                except:
                    exec(key + ' = None' , user_dict)
                    new_values[key] = 'None'
    if validated and special_question is None:
        if '_order_changes' in post_data:
            orderChanges = json.loads(post_data['_order_changes'])
            for tableName, changes in orderChanges.items():
                tableName = myb64unquote(tableName)
                #if STRICT_MODE and tableName not in authorized_fields:
                #    raise DAError("The variable " + repr(tableName) + " was not in the allowed fields, which were " + repr(authorized_fields))
                if illegal_variable_name(tableName):
                    error_messages.append(("error", "Error: Invalid character in table reorder: " + str(tableName)))
                    continue
                for item in changes:
                    if not (isinstance(item, list) and len(item) == 2 and isinstance(item[0], int) and isinstance(item[1], int)):
                        error_messages.append(("error", "Error: Invalid row number in table reorder: " + str(tableName) + " " + str(item)))
                        break
                exec(tableName + '._reorder(' + ', '.join([repr(item) for item in changes]) + ')', user_dict)
        inline_files_processed = []
        if '_files_inline' in post_data:
            fileDict = json.loads(myb64unquote(post_data['_files_inline']))
            if not isinstance(fileDict, dict):
                raise DAError("inline files was not a dict")
            file_fields = fileDict['keys']
            has_invalid_fields = False
            should_assemble_now = False
            empty_file_vars = set()
            for orig_file_field in file_fields:
                if orig_file_field in known_varnames:
                    orig_file_field = known_varnames[orig_file_field]
                if orig_file_field not in visible_fields:
                    empty_file_vars.add(orig_file_field)
                try:
                    file_field = from_safeid(orig_file_field)
                except:
                    error_messages.append(("error", "Error: Invalid file_field: " + orig_file_field))
                    break
                if STRICT_MODE and file_field not in authorized_fields:
                    raise DAError("The variable " + repr(file_field) + " was not in the allowed fields, which were " + repr(authorized_fields))
                if illegal_variable_name(file_field):
                    has_invalid_fields = True
                    error_messages.append(("error", "Error: Invalid character in file_field: " + str(file_field)))
                    break
                if key_requires_preassembly.search(file_field):
                    should_assemble_now = True
            if not has_invalid_fields:
                initial_string = 'import docassemble.base.util'
                try:
                    exec(initial_string, user_dict)
                except Exception as errMess:
                    error_messages.append(("error", "Error: " + str(errMess)))
                if should_assemble_now and not already_assembled:
                    interview.assemble(user_dict, interview_status)
                for orig_file_field_raw in file_fields:
                    if orig_file_field_raw in known_varnames:
                        orig_file_field_raw = known_varnames[orig_file_field_raw]
                    set_empty = bool(orig_file_field_raw not in visible_fields)
                    if not validated:
                        break
                    orig_file_field = orig_file_field_raw
                    var_to_store = orig_file_field_raw
                    if orig_file_field not in fileDict['values'] and len(known_varnames):
                        for key, val in known_varnames_visible.items():
                            if val == orig_file_field_raw:
                                orig_file_field = key
                                var_to_store = val
                                break
                    if orig_file_field in fileDict['values']:
                        the_files = fileDict['values'][orig_file_field]
                        if the_files:
                            files_to_process = []
                            for the_file in the_files:
                                temp_file = tempfile.NamedTemporaryFile(prefix="datemp", delete=False)
                                start_index = 0
                                char_index = 0
                                for char in the_file['content']:
                                    char_index += 1
                                    if char == ',':
                                        start_index = char_index
                                        break
                                temp_file.write(codecs.decode(bytearray(the_file['content'][start_index:], encoding='utf-8'), 'base64'))
                                temp_file.close()
                                filename = secure_filename(the_file['name'])
                                extension, mimetype = get_ext_and_mimetype(filename)
                                try:
                                    img = Image.open(temp_file.name)
                                    the_format = img.format.lower()
                                    the_format = re.sub(r'jpeg', 'jpg', the_format)
                                except:
                                    the_format = extension
                                    logmessage("Could not read file type from file " + str(filename))
                                if the_format != extension:
                                    filename = re.sub(r'\.[^\.]+$', '', filename) + '.' + the_format
                                    extension, mimetype = get_ext_and_mimetype(filename)
                                file_number = get_new_file_number(user_code, filename, yaml_file_name=yaml_filename)
                                saved_file = SavedFile(file_number, extension=extension, fix=True, should_not_exist=True)
                                process_file(saved_file, temp_file.name, mimetype, extension)
                                files_to_process.append((filename, file_number, mimetype, extension))
                            try:
                                file_field = from_safeid(var_to_store)
                            except:
                                error_messages.append(("error", "Error: Invalid file_field: " + str(var_to_store)))
                                break
                            if STRICT_MODE and file_field not in authorized_fields:
                                raise DAError("The variable " + repr(file_field) + " was not in the allowed fields, which were " + repr(authorized_fields))
                            if illegal_variable_name(file_field):
                                error_messages.append(("error", "Error: Invalid character in file_field: " + str(file_field)))
                                break
                            file_field_tr = sub_indices(file_field, user_dict)
                            if len(files_to_process) > 0:
                                elements = []
                                indexno = 0
                                for (filename, file_number, mimetype, extension) in files_to_process:
                                    elements.append("docassemble.base.util.DAFile(" + repr(file_field_tr + "[" + str(indexno) + "]") + ", filename=" + repr(filename) + ", number=" + str(file_number) + ", make_pngs=True, mimetype=" + repr(mimetype) + ", extension=" + repr(extension) + ")")
                                    indexno += 1
                                the_file_list = "docassemble.base.util.DAFileList(" + repr(file_field_tr) + ", elements=[" + ", ".join(elements) + "])"
                                if orig_file_field in field_numbers and the_question is not None and len(the_question.fields) > field_numbers[orig_file_field]:
                                    the_field = the_question.fields[field_numbers[orig_file_field]]
                                    add_permissions_for_field(the_field, interview_status, files_to_process)
                                    if hasattr(the_field, 'validate'):
                                        the_key = orig_file_field
                                        the_func = eval(the_field.validate['compute'], user_dict)
                                        try:
                                            the_result = the_func(eval(the_file_list))
                                            if not the_result:
                                                field_error[the_key] = word("Please enter a valid value.")
                                                validated = False
                                                break
                                        except Exception as errstr:
                                            field_error[the_key] = str(errstr)
                                            validated = False
                                            break
                                the_string = file_field + " = " + the_file_list
                                inline_files_processed.append(file_field)
                            else:
                                the_string = file_field + " = None"
                            key_to_orig_key[file_field] = orig_file_field
                            process_set_variable(file_field, user_dict, vars_set, old_values)
                            try:
                                exec(the_string, user_dict)
                                changed = True
                            except Exception as errMess:
                                try:
                                    sys.stderr.write("Error: " + errMess.__class__.__name__ + ": " + str(errMess) + " after trying to run " + the_string + "\n")
                                except:
                                    pass
                                error_messages.append(("error", "Error: " + errMess.__class__.__name__ + ": " + str(errMess)))
                    else:
                        try:
                            file_field = from_safeid(var_to_store)
                        except:
                            error_messages.append(("error", "Error: Invalid file_field: " + str(var_to_store)))
                            break
                        if STRICT_MODE and file_field not in authorized_fields:
                            raise DAError("The variable " + repr(file_field) + " was not in the allowed fields, which were " + repr(authorized_fields))
                        if illegal_variable_name(file_field):
                            error_messages.append(("error", "Error: Invalid character in file_field: " + str(file_field)))
                            break
                        the_string = file_field + " = None"
                        key_to_orig_key[file_field] = orig_file_field
                        process_set_variable(file_field, user_dict, vars_set, old_values)
                        try:
                            exec(the_string, user_dict)
                            changed = True
                        except Exception as errMess:
                            sys.stderr.write("Error: " + errMess.__class__.__name__ + ": " + str(errMess) + " after running " + the_string + "\n")
                            error_messages.append(("error", "Error: " + errMess.__class__.__name__ + ": " + str(errMess)))
        if '_files' in post_data or (STRICT_MODE and (not disregard_input) and len(field_info['files']) > 0):
            if STRICT_MODE:
                file_fields = field_info['files']
            else:
                file_fields = json.loads(myb64unquote(post_data['_files']))
            has_invalid_fields = False
            should_assemble_now = False
            empty_file_vars = set()
            for orig_file_field in file_fields:
                if orig_file_field not in raw_visible_fields:
                    continue
                if orig_file_field in known_varnames:
                    orig_file_field = known_varnames[orig_file_field]
                if orig_file_field not in visible_fields:
                    empty_file_vars.add(orig_file_field)
                try:
                    file_field = from_safeid(orig_file_field)
                except:
                    error_messages.append(("error", "Error: Invalid file_field: " + str(orig_file_field)))
                    break
                if STRICT_MODE and file_field not in authorized_fields:
                    raise DAError("The variable " + repr(file_field) + " was not in the allowed fields, which were " + repr(authorized_fields))
                if illegal_variable_name(file_field):
                    has_invalid_fields = True
                    error_messages.append(("error", "Error: Invalid character in file_field: " + str(file_field)))
                    break
                if key_requires_preassembly.search(file_field):
                    should_assemble_now = True
                key_to_orig_key[file_field] = orig_file_field
            if not has_invalid_fields:
                initial_string = 'import docassemble.base.util'
                try:
                    exec(initial_string, user_dict)
                except Exception as errMess:
                    error_messages.append(("error", "Error: " + str(errMess)))
                if not already_assembled:
                    interview.assemble(user_dict, interview_status)
                for orig_file_field_raw in file_fields:
                    if orig_file_field_raw not in raw_visible_fields:
                        continue
                    if orig_file_field_raw in known_varnames:
                        orig_file_field_raw = known_varnames[orig_file_field_raw]
                    if orig_file_field_raw not in visible_fields:
                        continue
                    if not validated:
                        break
                    orig_file_field = orig_file_field_raw
                    var_to_store = orig_file_field_raw
                    if (orig_file_field not in request.files or request.files[orig_file_field].filename == "") and len(known_varnames):
                        for key, val in known_varnames_visible.items():
                            if val == orig_file_field_raw:
                                orig_file_field = key
                                var_to_store = val
                                break
                    if orig_file_field in request.files and request.files[orig_file_field].filename != "":
                        the_files = request.files.getlist(orig_file_field)
                        if the_files:
                            files_to_process = []
                            for the_file in the_files:
                                if is_ajax:
                                    return_fake_html = True
                                filename = secure_filename(the_file.filename)
                                file_number = get_new_file_number(user_code, filename, yaml_file_name=yaml_filename)
                                extension, mimetype = get_ext_and_mimetype(filename)
                                saved_file = SavedFile(file_number, extension=extension, fix=True, should_not_exist=True)
                                temp_file = tempfile.NamedTemporaryFile(prefix="datemp", suffix='.' + extension, delete=False)
                                the_file.save(temp_file.name)
                                process_file(saved_file, temp_file.name, mimetype, extension)
                                files_to_process.append((filename, file_number, mimetype, extension))
                            try:
                                file_field = from_safeid(var_to_store)
                            except:
                                error_messages.append(("error", "Error: Invalid file_field: " + str(var_to_store)))
                                break
                            if STRICT_MODE and file_field not in authorized_fields:
                                raise DAError("The variable " + repr(file_field) + " was not in the allowed fields, which were " + repr(authorized_fields))
                            if illegal_variable_name(file_field):
                                error_messages.append(("error", "Error: Invalid character in file_field: " + str(file_field)))
                                break
                            file_field_tr = sub_indices(file_field, user_dict)
                            if len(files_to_process) > 0:
                                elements = []
                                indexno = 0
                                for (filename, file_number, mimetype, extension) in files_to_process:
                                    elements.append("docassemble.base.util.DAFile(" + repr(file_field_tr + '[' + str(indexno) + ']') + ", filename=" + repr(filename) + ", number=" + str(file_number) + ", make_pngs=True, mimetype=" + repr(mimetype) + ", extension=" + repr(extension) + ")")
                                    indexno += 1
                                the_file_list = "docassemble.base.util.DAFileList(" + repr(file_field_tr) + ", elements=[" + ", ".join(elements) + "])"
                                if orig_file_field in field_numbers and the_question is not None and len(the_question.fields) > field_numbers[orig_file_field]:
                                    the_field = the_question.fields[field_numbers[orig_file_field]]
                                    add_permissions_for_field(the_field, interview_status, files_to_process)
                                    if hasattr(the_question.fields[field_numbers[orig_file_field]], 'validate'):
                                        the_key = orig_file_field
                                        the_func = eval(the_question.fields[field_numbers[orig_file_field]].validate['compute'], user_dict)
                                        try:
                                            the_result = the_func(eval(the_file_list))
                                            if not the_result:
                                                field_error[the_key] = word("Please enter a valid value.")
                                                validated = False
                                                break
                                        except Exception as errstr:
                                            field_error[the_key] = str(errstr)
                                            validated = False
                                            break
                                the_string = file_field + " = " + the_file_list
                            else:
                                the_string = file_field + " = None"
                            process_set_variable(file_field, user_dict, vars_set, old_values)
                            if validated:
                                try:
                                    exec(the_string, user_dict)
                                    changed = True
                                except Exception as errMess:
                                    sys.stderr.write("Error: " + errMess.__class__.__name__ + ": " + str(errMess) + "after running " + the_string + "\n")
                                    error_messages.append(("error", "Error: " + errMess.__class__.__name__ + ": " + str(errMess)))
                    else:
                        try:
                            file_field = from_safeid(var_to_store)
                        except:
                            error_messages.append(("error", "Error: Invalid file_field: " + str(var_to_store)))
                            break
                        if file_field in inline_files_processed:
                            continue
                        if STRICT_MODE and file_field not in authorized_fields:
                            raise DAError("The variable " + repr(file_field) + " was not in the allowed fields, which were " + repr(authorized_fields))
                        if illegal_variable_name(file_field):
                            error_messages.append(("error", "Error: Invalid character in file_field: " + str(file_field)))
                            break
                        the_string = file_field + " = None"
                        process_set_variable(file_field, user_dict, vars_set, old_values)
                        try:
                            exec(the_string, user_dict)
                            changed = True
                        except Exception as errMess:
                            sys.stderr.write("Error: " + errMess.__class__.__name__ + ": " + str(errMess) + "after running " + the_string + "\n")
                            error_messages.append(("error", "Error: " + errMess.__class__.__name__ + ": " + str(errMess)))
        if validated:
            if 'informed' in request.form:
                user_dict['_internal']['informed'][the_user_id] = {}
                for key in request.form['informed'].split(','):
                    user_dict['_internal']['informed'][the_user_id][key] = 1
            if changed and '_question_name' in post_data and post_data['_question_name'] not in user_dict['_internal']['answers']:
                try:
                    interview.questions_by_name[post_data['_question_name']].mark_as_answered(user_dict)
                except:
                    logmessage("index: question name could not be found")
            if ('_event' in post_data or (STRICT_MODE and (not disregard_input) and field_info['orig_sought'] is not None)) and 'event_stack' in user_dict['_internal']:
                if STRICT_MODE:
                    events_list = [field_info['orig_sought']]
                else:
                    events_list = json.loads(myb64unquote(post_data['_event']))
                if len(events_list) > 0:
                    session_uid = interview_status.current_info['user']['session_uid']
                    if session_uid in user_dict['_internal']['event_stack'] and len(user_dict['_internal']['event_stack'][session_uid]):
                        for event_name in events_list:
                            if user_dict['_internal']['event_stack'][session_uid][0]['action'] == event_name:
                                user_dict['_internal']['event_stack'][session_uid].pop(0)
                                if 'action' in interview_status.current_info and interview_status.current_info['action'] == event_name:
                                    del interview_status.current_info['action']
                                    if 'arguments' in interview_status.current_info:
                                        del interview_status.current_info['arguments']
                                break
                            if len(user_dict['_internal']['event_stack'][session_uid]) == 0:
                                break
            for var_name in list(vars_set):
                vars_set.add(sub_indices(var_name, user_dict))
            if len(vars_set) > 0 and 'event_stack' in user_dict['_internal']:
                session_uid = interview_status.current_info['user']['session_uid']
                popped = True
                while popped:
                    popped = False
                    if session_uid in user_dict['_internal']['event_stack'] and len(user_dict['_internal']['event_stack'][session_uid]):
                        for var_name in vars_set:
                            if user_dict['_internal']['event_stack'][session_uid][0]['action'] == var_name:
                                popped = True
                                user_dict['_internal']['event_stack'][session_uid].pop(0)
                            if len(user_dict['_internal']['event_stack'][session_uid]) == 0:
                                break
        else:
            steps, user_dict, is_encrypted = fetch_user_dict(user_code, yaml_filename, secret=secret)
    else:
        steps, user_dict, is_encrypted = fetch_user_dict(user_code, yaml_filename, secret=secret)
    if validated and special_question is None:
        if '_collect_delete' in post_data and list_collect_list is not None:
            to_delete = json.loads(post_data['_collect_delete'])
            is_ok = True
            for item in to_delete:
                if not isinstance(item, int):
                    is_ok = False
            if is_ok:
                if not illegal_variable_name(list_collect_list):
                    exec(list_collect_list + ' ._remove_items_by_number(' + ', '.join(map(lambda y: str(y), to_delete)) + ')' , user_dict)
                    changed = True
        if '_collect' in post_data and list_collect_list is not None:
            collect = json.loads(myb64unquote(post_data['_collect']))
            # if STRICT_MODE and collect['list'] not in authorized_fields:
            #     raise DAError("The variable " + repr(collect['list']) + " was not in the allowed fields, which were " + repr(authorized_fields))
            #if not illegal_variable_name(collect['list']):
            if collect['function'] == 'add':
                add_action_to_stack(interview_status, user_dict, '_da_list_add', {'list': list_collect_list})
        if list_collect_list is not None:
            exec(list_collect_list + '._disallow_appending()' , user_dict)
        if the_question is not None and the_question.validation_code:
            try:
                exec(the_question.validation_code, user_dict)
            except Exception as validation_error:
                the_error_message = str(validation_error)
                logmessage("index: exception during validation: " + the_error_message)
                if the_error_message == '':
                    the_error_message = word("Please enter a valid value.")
                if isinstance(validation_error, DAValidationError) and isinstance(validation_error.field, str) and validation_error.field in key_to_orig_key:
                    field_error[key_to_orig_key[validation_error.field]] = the_error_message
                else:
                    error_messages.append(("error", the_error_message))
                validated = False
                steps, user_dict, is_encrypted = fetch_user_dict(user_code, yaml_filename, secret=secret)
    if validated:
        for var_name in vars_set:
            if var_name in interview.invalidation_todo or var_name in interview.onchange_todo:
                interview.invalidate_dependencies(var_name, user_dict, old_values)
            try:
                del user_dict['_internal']['dirty'][var_name]
            except:
                pass
    if action is not None:
        interview_status.current_info.update(action)
    interview.assemble(user_dict, interview_status, old_user_dict, force_question=special_question)
    current_language = docassemble.base.functions.get_language()
    session['language'] = current_language
    if not interview_status.can_go_back:
        user_dict['_internal']['steps_offset'] = steps
    if was_new:
        docassemble.base.functions.this_thread.misc['save_status'] = 'overwrite'
    if not changed and url_args_changed:
        changed = True
        validated = True
    if interview_status.question.question_type == "restart":
        manual_checkout(manual_filename=yaml_filename)
        url_args = user_dict['url_args']
        referer = user_dict['_internal'].get('referer', None)
        user_dict = fresh_dictionary()
        user_dict['url_args'] = url_args
        user_dict['_internal']['referer'] = referer
        interview_status = docassemble.base.parse.InterviewStatus(current_info=current_info(yaml=yaml_filename, req=request, interface=the_interface, session_info=session_info, secret=secret, device_id=device_id))
        reset_user_dict(user_code, yaml_filename)
        if 'visitor_secret' not in request.cookies:
            save_user_dict_key(user_code, yaml_filename)
            update_session(yaml_filename, uid=user_code, key_logged=True)
        steps = 1
        changed = False
        interview.assemble(user_dict, interview_status)
    elif interview_status.question.question_type == "new_session":
        manual_checkout(manual_filename=yaml_filename)
        url_args = user_dict['url_args']
        referer = user_dict['_internal'].get('referer', None)
        interview_status = docassemble.base.parse.InterviewStatus(current_info=current_info(yaml=yaml_filename, req=request, interface=the_interface, session_info=session_info, secret=secret, device_id=device_id))
        release_lock(user_code, yaml_filename)
        user_code, user_dict = reset_session(yaml_filename, secret)
        user_dict['url_args'] = url_args
        user_dict['_internal']['referer'] = referer
        if 'visitor_secret' not in request.cookies:
            save_user_dict_key(user_code, yaml_filename)
            update_session(yaml_filename, uid=user_code, key_logged=True)
        steps = 1
        changed = False
        interview.assemble(user_dict, interview_status)
    title_info = interview_status.question.interview.get_title(user_dict, status=interview_status, converter=lambda content, part: title_converter(content, part, interview_status))
    save_status = docassemble.base.functions.this_thread.misc.get('save_status', 'new')
    if interview_status.question.question_type == "exit":
        manual_checkout(manual_filename=yaml_filename)
        reset_user_dict(user_code, yaml_filename)
        delete_session_for_interview(i=yaml_filename)
        release_lock(user_code, yaml_filename)
        sys.stderr.write("Redirecting because of an exit.\n")
        if interview_status.questionText != '':
            response = do_redirect(interview_status.questionText, is_ajax, is_json, js_target)
        else:
            response = do_redirect(title_info.get('exit url', None) or exit_page, is_ajax, is_json, js_target)
        if return_fake_html:
            fake_up(response, current_language)
        if response_wrapper:
            response_wrapper(response)
        return response
    if interview_status.question.question_type in ("exit_logout", "logout"):
        manual_checkout(manual_filename=yaml_filename)
        if interview_status.question.question_type == "exit_logout":
            reset_user_dict(user_code, yaml_filename)
        release_lock(user_code, yaml_filename)
        delete_session_info()
        sys.stderr.write("Redirecting because of a logout.\n")
        if interview_status.questionText != '':
            response = do_redirect(interview_status.questionText, is_ajax, is_json, js_target)
        else:
            response = do_redirect(title_info.get('exit url', None) or exit_page, is_ajax, is_json, js_target)
        if current_user.is_authenticated:
            docassemble_flask_user.signals.user_logged_out.send(current_app._get_current_object(), user=current_user)
            logout_user()
        delete_session_info()
        session.clear()
        response.set_cookie('remember_token', '', expires=0)
        response.set_cookie('visitor_secret', '', expires=0)
        response.set_cookie('secret', '', expires=0)
        response.set_cookie('session', '', expires=0)
        if return_fake_html:
            fake_up(response, current_language)
        return response
    will_save = True
    if interview_status.question.question_type == "refresh":
        release_lock(user_code, yaml_filename)
        response = do_refresh(is_ajax, yaml_filename)
        if return_fake_html:
            fake_up(response, current_language)
        if response_wrapper:
            response_wrapper(response)
        return response
    if interview_status.question.question_type == "signin":
        release_lock(user_code, yaml_filename)
        sys.stderr.write("Redirecting because of a signin.\n")
        response = do_redirect(url_for('user.login', next=url_for('index', i=yaml_filename, session=user_code)), is_ajax, is_json, js_target)
        if return_fake_html:
            fake_up(response, current_language)
        if response_wrapper:
            response_wrapper(response)
        return response
    if interview_status.question.question_type == "register":
        release_lock(user_code, yaml_filename)
        sys.stderr.write("Redirecting because of a register.\n")
        response = do_redirect(url_for('user.register', next=url_for('index', i=yaml_filename, session=user_code)), is_ajax, is_json, js_target)
        if return_fake_html:
            fake_up(response, current_language)
        if response_wrapper:
            response_wrapper(response)
        return response
    if interview_status.question.question_type == "leave":
        release_lock(user_code, yaml_filename)
        sys.stderr.write("Redirecting because of a leave.\n")
        if interview_status.questionText != '':
            response = do_redirect(interview_status.questionText, is_ajax, is_json, js_target)
        else:
            response = do_redirect(title_info.get('exit url', None) or exit_page, is_ajax, is_json, js_target)
        if return_fake_html:
            fake_up(response, current_language)
        if response_wrapper:
            response_wrapper(response)
        return response
    if interview_status.question.interview.use_progress_bar and interview_status.question.progress is not None:
        if interview_status.question.progress == -1:
            user_dict['_internal']['progress'] = None
        elif user_dict['_internal']['progress'] is None or interview_status.question.interview.options.get('strict progress', False) or interview_status.question.progress > user_dict['_internal']['progress']:
            user_dict['_internal']['progress'] = interview_status.question.progress
    if interview_status.question.interview.use_navigation and interview_status.question.section is not None:
        user_dict['nav'].set_section(interview_status.question.section)
    if interview_status.question.question_type == "response":
        if is_ajax:
            release_lock(user_code, yaml_filename)
            response = jsonify(action='resubmit', csrf_token=generate_csrf())
            if return_fake_html:
                fake_up(response, current_language)
            if response_wrapper:
                response_wrapper(response)
            return response
        if hasattr(interview_status.question, 'response_code'):
            resp_code = interview_status.question.response_code
        else:
            resp_code = 200
        if hasattr(interview_status.question, 'all_variables'):
            if hasattr(interview_status.question, 'include_internal'):
                include_internal = interview_status.question.include_internal
            else:
                include_internal = False
            response_to_send = make_response(docassemble.base.functions.dict_as_json(user_dict, include_internal=include_internal).encode('utf-8'), resp_code)
        elif hasattr(interview_status.question, 'binaryresponse'):
            response_to_send = make_response(interview_status.question.binaryresponse, resp_code)
        else:
            response_to_send = make_response(interview_status.questionText.encode('utf-8'), resp_code)
        response_to_send.headers['Content-Type'] = interview_status.extras['content_type']
    elif interview_status.question.question_type == "sendfile":
        if is_ajax:
            release_lock(user_code, yaml_filename)
            response = jsonify(action='resubmit', csrf_token=generate_csrf())
            if return_fake_html:
                fake_up(response, current_language)
            if response_wrapper:
                response_wrapper(response)
            return response
        if interview_status.question.response_file is not None:
            the_path = interview_status.question.response_file.path()
        else:
            logmessage("index: could not send file because the response was None")
            return ('File not found', 404)
        if not os.path.isfile(the_path):
            logmessage("index: could not send file because file (" + the_path + ") not found")
            return ('File not found', 404)
        response_to_send = send_file(the_path, mimetype=interview_status.extras['content_type'])
        response_to_send.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
    elif interview_status.question.question_type == "redirect":
        sys.stderr.write("Redirecting because of a redirect.\n")
        response_to_send = do_redirect(interview_status.questionText, is_ajax, is_json, js_target)
    else:
        response_to_send = None
    if (not interview_status.followed_mc) and len(user_dict['_internal']['answers']):
        user_dict['_internal']['answers'].clear()
    if not validated:
        changed = False
    if changed and validated:
        if save_status == 'new':
            steps += 1
            user_dict['_internal']['steps'] = steps
    if action and not changed:
        changed = True
        if save_status == 'new':
            steps += 1
            user_dict['_internal']['steps'] = steps
    if changed and interview_status.question.interview.use_progress_bar and interview_status.question.progress is None and save_status == 'new':
        advance_progress(user_dict, interview)
    title_info = interview_status.question.interview.get_title(user_dict, status=interview_status, converter=lambda content, part: title_converter(content, part, interview_status))
    if save_status != 'ignore':
        if save_status == 'overwrite':
            changed = False
        save_user_dict(user_code, user_dict, yaml_filename, secret=secret, changed=changed, encrypt=encrypted, steps=steps)
        if user_dict.get('multi_user', False) is True and encrypted is True:
            encrypted = False
            update_session(yaml_filename, encrypted=encrypted)
            decrypt_session(secret, user_code=user_code, filename=yaml_filename)
        if user_dict.get('multi_user', False) is False and encrypted is False:
            encrypt_session(secret, user_code=user_code, filename=yaml_filename)
            encrypted = True
            update_session(yaml_filename, encrypted=encrypted)
    if response_to_send is not None:
        release_lock(user_code, yaml_filename)
        if return_fake_html:
            fake_up(response_to_send, current_language)
        if response_wrapper:
            response_wrapper(response_to_send)
        return response_to_send
    messages = get_flashed_messages(with_categories=True) + error_messages
    if messages and len(messages):
        notification_interior = ''
        for classname, message in messages:
            if classname == 'error':
                classname = 'danger'
            notification_interior += NOTIFICATION_MESSAGE % (classname, str(message))
        flash_content = NOTIFICATION_CONTAINER % (notification_interior,)
    else:
        flash_content = ''
    if 'reload_after' in interview_status.extras:
        reload_after = 1000 * int(interview_status.extras['reload_after'])
    else:
        reload_after = 0
    allow_going_back = bool(interview_status.question.can_go_back and (steps - user_dict['_internal']['steps_offset']) > 1)
    if hasattr(interview_status.question, 'id'):
        question_id = interview_status.question.id
    else:
        question_id = None
    question_id_dict = dict(id=question_id)
    if interview_status.question.interview.options.get('analytics on', True):
        if 'segment' in interview_status.extras:
            question_id_dict['segment'] = interview_status.extras['segment']
        if 'ga_id' in interview_status.extras:
            question_id_dict['ga'] = interview_status.extras['ga_id']
    append_script_urls = []
    append_javascript = ''
    if not is_ajax:
        scripts = standard_scripts(interview_language=current_language) + additional_scripts(interview_status, yaml_filename)
        if is_js:
            append_javascript += additional_scripts(interview_status, yaml_filename, as_javascript=True)
        if 'javascript' in interview_status.question.interview.external_files:
            for packageref, fileref in interview_status.question.interview.external_files['javascript']:
                the_url = get_url_from_file_reference(fileref, _package=packageref)
                if the_url is not None:
                    scripts += "\n" + '    <script src="' + get_url_from_file_reference(fileref, _package=packageref) + '"></script>'
                    if is_js:
                        append_script_urls.append(get_url_from_file_reference(fileref, _package=packageref))
                else:
                    logmessage("index: could not find javascript file " + str(fileref))
        if interview_status.question.checkin is not None:
            do_action = json.dumps(interview_status.question.checkin)
        else:
            do_action = 'null'
        chat_available = user_dict['_internal']['livehelp']['availability']
        chat_mode = user_dict['_internal']['livehelp']['mode']
        if chat_available == 'unavailable':
            chat_status = 'off'
            update_session(yaml_filename, chatstatus='off')
        elif chat_available == 'observeonly':
            chat_status = 'observeonly'
            update_session(yaml_filename, chatstatus='observeonly')
        else:
            chat_status = session_info['chatstatus']
        if chat_status in ('ready', 'on'):
            chat_status = 'ringing'
            update_session(yaml_filename, chatstatus='ringing')
        if chat_status != 'off':
            send_changes = 'true'
        else:
            if do_action != 'null':
                send_changes = 'true'
            else:
                send_changes = 'false'
        if current_user.is_authenticated:
            user_id_string = str(current_user.id)
            if current_user.has_role('admin', 'developer', 'advocate'):
                is_user = 'false'
            else:
                is_user = 'true'
        else:
            user_id_string = 't' + str(session['tempuser'])
            is_user = 'true'
        if r.get('da:control:uid:' + str(user_code) + ':i:' + str(yaml_filename) + ':userid:' + str(the_user_id)) is not None:
            being_controlled = 'true'
        else:
            being_controlled = 'false'
        if debug_mode:
            debug_readability_help = """
            $("#dareadability-help").show();
            $("#dareadability-question").hide();
"""
            debug_readability_question = """
            $("#dareadability-help").hide();
            $("#dareadability-question").show();
"""
        else:
            debug_readability_help = ''
            debug_readability_question = ''
        if interview_status.question.interview.force_fullscreen is True or (re.search(r'mobile', str(interview_status.question.interview.force_fullscreen).lower()) and is_mobile_or_tablet()):
            forceFullScreen = """
          if (data.steps > 1 && window != top) {
            top.location.href = location.href;
            return;
          }
"""
        else:
            forceFullScreen = ''
        the_checkin_interval = interview_status.question.interview.options.get('checkin interval', CHECKIN_INTERVAL)
        if interview_status.question.interview.options.get('analytics on', True):
            if ga_configured:
                ga_id = google_config.get('analytics id')
            else:
                ga_id = None
            if 'segment id' in daconfig:
                segment_id = daconfig['segment id']
            else:
                segment_id = None
        else:
            ga_id = None
            segment_id = None
        page_sep = "#page"
        if refer is None:
            location_bar = url_for('index', **index_params)
        elif refer[0] in ('start', 'run'):
            location_bar = url_for('run_interview_in_package', package=refer[1], filename=refer[2])
            page_sep = "#/"
        elif refer[0] in ('start_dispatch', 'run_dispatch'):
            location_bar = url_for('run_interview', dispatch=refer[1])
            page_sep = "#/"
        elif refer[0] in ('start_directory', 'run_directory'):
            location_bar = url_for('run_interview_in_package_directory', package=refer[1], directory=refer[2], filename=refer[3])
            page_sep = "#/"
        else:
            location_bar = None
            for k, v in daconfig['dispatch'].items():
                if v == yaml_filename:
                    location_bar = url_for('run_interview', dispatch=k)
                    page_sep = "#/"
                    break
            if location_bar is None:
                location_bar = url_for('index', **index_params)
        index_params_external = copy.copy(index_params)
        index_params_external['_external'] = True
        the_js = """\
      if (typeof($) == 'undefined'){
        var $ = jQuery.noConflict();
      }
      var daMapInfo = null;
      var daWhichButton = null;
      var daSocket = null;
      var daChatHistory = [];
      var daCheckinCode = null;
      var daCheckingIn = 0;
      var daShowingHelp = 0;
      var daIframeEmbed;
      if ( window.location !== window.parent.location ) {
        daIframeEmbed = true;
      }
      else {
        daIframeEmbed = false;
      }
      var daJsEmbed = """ + (json.dumps(js_target) if is_js else 'false') + """;
      var daAllowGoingBack = """ + ('true' if allow_going_back else 'false') + """;
      var daSteps = """ + str(steps) + """;
      var daIsUser = """ + is_user + """;
      var daChatStatus = """ + json.dumps(chat_status) + """;
      var daChatAvailable = """ + json.dumps(chat_available) + """;
      var daChatPartnersAvailable = 0;
      var daPhoneAvailable = false;
      var daChatMode = """ + json.dumps(chat_mode) + """;
      var daSendChanges = """ + send_changes + """;
      var daInitialized = false;
      var daNotYetScrolled = true;
      var daBeingControlled = """ + being_controlled + """;
      var daInformedChanged = false;
      var daInformed = """ + json.dumps(user_dict['_internal']['informed'].get(user_id_string, {})) + """;
      var daShowingSpinner = false;
      var daSpinnerTimeout = null;
      var daSubmitter = null;
      var daUsingGA = """ + ("true" if ga_id is not None else 'false') + """;
      var daUsingSegment = """ + ("true" if segment_id is not None else 'false') + """;
      var daDoAction = """ + do_action + """;
      var daQuestionID = """ + json.dumps(question_id_dict) + """;
      var daCsrf = """ + json.dumps(generate_csrf()) + """;
      var daShowIfInProcess = false;
      var daFieldsToSkip = ['_checkboxes', '_empties', '_ml_info', '_back_one', '_files', '_files_inline', '_question_name', '_the_image', '_save_as', '_success', '_datatypes', '_event', '_visible', '_tracker', '_track_location', '_varnames', '_next_action', '_next_action_to_set', 'ajax', 'json', 'informed', 'csrf_token', '_action', '_order_changes', '_collect', '_list_collect_list', '_null_question'];
      var daVarLookup = Object();
      var daVarLookupRev = Object();
      var daVarLookupMulti = Object();
      var daVarLookupRevMulti = Object();
      var daVarLookupSelect = Object();
      var daTargetDiv;
      var daComboBoxes = Object();
      var daGlobalEval = eval;
      var daInterviewUrl = """ + json.dumps(url_for('index', **index_params)) + """;
      var daLocationBar = """ + json.dumps(location_bar) + """;
      var daPostURL = """ + json.dumps(url_for('index', **index_params_external)) + """;
      var daYamlFilename = """ + json.dumps(yaml_filename) + """;
      var daFetchAcceptIncoming = false;
      var daFetchAjaxTimeout = null;
      var daFetchAjaxTimeoutRunning = null;
      var daFetchAjaxTimeoutFetchAfter = null;
      var daShowHideHappened = false;
      if (daJsEmbed){
        daTargetDiv = '#' + daJsEmbed;
      }
      else{
        daTargetDiv = "#dabody";
      }
      var daNotificationContainer = """ + json.dumps(NOTIFICATION_CONTAINER) + """;
      var daNotificationMessage = """ + json.dumps(NOTIFICATION_MESSAGE) + """;
      Object.defineProperty(String.prototype, "daSprintf", {
        value: function () {
          var args = Array.from(arguments),
            i = 0;
          function defaultNumber(iValue) {
            return iValue != undefined && !isNaN(iValue) ? iValue : "0";
          }
          function defaultString(iValue) {
            return iValue == undefined ? "" : "" + iValue;
          }
          return this.replace(
            /%%|%([+\\-])?([^1-9])?(\\d+)?(\\.\\d+)?([deEfhHioQqs])/g,
            function (match, sign, filler, scale, precision, type) {
              var strOut, space, value;
              var asNumber = false;
              if (match == "%%") return "%";
              if (i >= args.length) return match;
              value = args[i];
              while (Array.isArray(value)) {
                args.splice(i, 1);
                for (var j = i; value.length > 0; j++)
                  args.splice(j, 0, value.shift());
                value = args[i];
              }
              i++;
              if (filler == undefined) filler = " "; // default
              if (scale == undefined && !isNaN(filler)) {
                scale = filler;
                filler = " ";
              }
              if (sign == undefined) sign = "sqQ".indexOf(type) >= 0 ? "+" : "-"; // default
              if (scale == undefined) scale = 0; // default
              if (precision == undefined) precision = ".0"; // default
              scale = parseInt(scale);
              precision = parseInt(precision.substr(1));
              switch (type) {
                case "d":
                case "i":
                  // decimal integer
                  asNumber = true;
                  strOut = parseInt(defaultNumber(value));
                  if (precision > 0) strOut += "." + "0".repeat(precision);
                  break;
                case "e":
                case "E":
                  // float in exponential notation
                  asNumber = true;
                  strOut = parseFloat(defaultNumber(value));
                  if (precision == 0) strOut = strOut.toExponential();
                  else strOut = strOut.toExponential(precision);
                  if (type == "E") strOut = strOut.replace("e", "E");
                  break;
                case "f":
                  // decimal float
                  asNumber = true;
                  strOut = parseFloat(defaultNumber(value));
                  if (precision != 0) strOut = strOut.toFixed(precision);
                  break;
                case "o":
                case "h":
                case "H":
                  // Octal or Hexagesimal integer notation
                  strOut =
                    "\\\\" +
                    (type == "o" ? "0" : type) +
                    parseInt(defaultNumber(value)).toString(type == "o" ? 8 : 16);
                  break;
                case "q":
                  // single quoted string
                  strOut = "'" + defaultString(value) + "'";
                  break;
                case "Q":
                  // double quoted string
                  strOut = '"' + defaultString(value) + '"';
                  break;
                default:
                  // string
                  strOut = defaultString(value);
                  break;
              }
              if (typeof strOut != "string") strOut = "" + strOut;
              if ((space = strOut.length) < scale) {
                if (asNumber) {
                  if (sign == "-") {
                    if (strOut.indexOf("-") < 0)
                      strOut = filler.repeat(scale - space) + strOut;
                    else
                      strOut =
                        "-" +
                        filler.repeat(scale - space) +
                        strOut.replace("-", "");
                  } else {
                    if (strOut.indexOf("-") < 0)
                      strOut = "+" + filler.repeat(scale - space - 1) + strOut;
                    else
                      strOut =
                        "-" +
                        filler.repeat(scale - space) +
                        strOut.replace("-", "");
                  }
                } else {
                  if (sign == "-") strOut = filler.repeat(scale - space) + strOut;
                  else strOut = strOut + filler.repeat(scale - space);
                }
              } else if (asNumber && sign == "+" && strOut.indexOf("-") < 0)
                strOut = "+" + strOut;
              return strOut;
            }
          );
        },
      });
      Object.defineProperty(window, "daSprintf", {
        value: function (str, ...rest) {
          if (typeof str == "string")
            return String.prototype.daSprintf.apply(str, rest);
          return "";
        },
      });
      function daGoToAnchor(target){
        if (daJsEmbed){
          scrollTarget = $(target).first().position().top - 60;
        }
        else{
          scrollTarget = $(target).first().offset().top - 60;
        }
        if (scrollTarget != null){
          if (daJsEmbed){
            $(daTargetDiv).animate({
              scrollTop: scrollTarget
            }, 500);
          }
          else{
            $("html, body").animate({
              scrollTop: scrollTarget
            }, 500);
          }
        }
      }
      function dabtoa(str) {
        return window.btoa(str).replace(/[\\n=]/g, '');
      }
      function daatob(str) {
        return window.atob(str);
      }
      function getFields(){
        var allFields = [];
        for (var rawFieldName in daVarLookup){
          if (daVarLookup.hasOwnProperty(rawFieldName)){
            var fieldName = atob(rawFieldName);
            if (allFields.indexOf(fieldName) == -1){
              allFields.push(fieldName);
            }
          }
        }
        return allFields;
      }
      var daGetFields = getFields;
      function daAppendIfExists(fieldName, theArray){
        var elem = $("[name='" + fieldName + "']");
        if (elem.length > 0){
          for (var i = 0; i < theArray.length; ++i){
            if (theArray[i] == elem[0]){
              return;
            }
          }
          theArray.push(elem[0]);
        }
      }
      function getField(fieldName, notInDiv){
        if (daVarLookupSelect[fieldName]){
          var n = daVarLookupSelect[fieldName].length;
          for (var i = 0; i < n; ++i){
            var elem = daVarLookupSelect[fieldName][i].select;
            if (!$(elem).prop('disabled')){
              var showifParents = $(elem).parents(".dajsshowif,.dashowif");
              if (showifParents.length == 0 || $(showifParents[0]).data("isVisible") == '1'){
                if (notInDiv && $.contains(notInDiv, elem)){
                  continue;
                }
                return elem;
              }
            }
          }
        }
        var fieldNameEscaped = dabtoa(fieldName);
        var possibleElements = [];
        daAppendIfExists(fieldNameEscaped, possibleElements);
        if (daVarLookupMulti.hasOwnProperty(fieldNameEscaped)){
          for (var i = 0; i < daVarLookupMulti[fieldNameEscaped].length; ++i){
            daAppendIfExists(daVarLookupMulti[fieldNameEscaped][i], possibleElements);
          }
        }
        var returnVal = null;
        for (var i = 0; i < possibleElements.length; ++i){
          if (!$(possibleElements[i]).prop('disabled')){
            var showifParents = $(possibleElements[i]).parents(".dajsshowif,.dashowif");
            if (showifParents.length == 0 || $(showifParents[0]).data("isVisible") == '1'){
              if (notInDiv && $.contains(notInDiv, possibleElements[i])){
                continue;
              }
              returnVal = possibleElements[i];
            }
          }
        }
        return returnVal;
      }
      var daGetField = getField;
      function setField(fieldName, val){
        var elem = daGetField(fieldName);
        if (elem == null){
          console.log('setField: reference to non-existent field ' + fieldName);
          return;
        }
        if ($(elem).attr('type') == "checkbox"){
          if (val){
            if ($(elem).prop('checked') != true){
              $(elem).prop('checked', true);
              $(elem).trigger('change');
            }
          }
          else{
            if ($(elem).prop('checked') != false){
              $(elem).prop('checked', false);
              $(elem).trigger('change');
            }
          }
        }
        else if ($(elem).attr('type') == "radio"){
          var fieldNameEscaped = $(elem).attr('name').replace(/(:|\.|\[|\]|,|=)/g, "\\\\$1");
          var wasSet = false;
          $("input[name='" + fieldNameEscaped + "']").each(function(){
            if ($(this).val() == val){
              if ($(this).prop('checked') != true){
                $(this).prop('checked', true);
                $(this).trigger('change');
              }
              wasSet = true;
              return false;
            }
          });
          if (!wasSet){
            console.log('setField: could not set radio button ' + fieldName + ' to ' + val);
          }
        }
        else{
          if ($(elem).val() != val){
            $(elem).val(val);
            $(elem).trigger('change');
          }
        }
      }
      var daSetField = setField;
      function val(fieldName){
        var elem = daGetField(fieldName);
        if (elem == null){
          return null;
        }
        if ($(elem).attr('type') == "checkbox"){
          if ($(elem).prop('checked')){
            theVal = true;
          }
          else{
            theVal = false;
          }
        }
        else if ($(elem).attr('type') == "radio"){
          var fieldNameEscaped = $(elem).attr('name').replace(/(:|\.|\[|\]|,|=)/g, "\\\\$1");
          theVal = $("input[name='" + fieldNameEscaped + "']:checked").val();
          if (typeof(theVal) == 'undefined'){
            theVal = null;
          }
          else{
            if (theVal == 'True'){
              theVal = true;
            }
            else if (theVal == 'False'){
              theVal = false;
            }
          }
        }
        else if ($(elem).prop('tagName') == "SELECT" && $(elem).hasClass('damultiselect') && daVarLookupSelect[fieldName]){
          var n = daVarLookupSelect[fieldName].length;
          for (var i = 0; i < n; ++i){
            if (daVarLookupSelect[fieldName][i].select === elem){
              return $(daVarLookupSelect[fieldName][i].option).prop('selected');
            }
          }
        }
        else{
          theVal = $(elem).val();
        }
        return theVal;
      }
      var da_val = val;
      function daFormAsJSON(){
        var formData = $("#daform").serializeArray();
        var data = Object();
        var n = formData.length;
        for (var i = 0; i < n; ++i){
          var key = formData[i]['name'];
          var val = formData[i]['value'];
          if ($.inArray(key, daFieldsToSkip) != -1 || key.indexOf('_ignore') == 0){
            continue;
          }
          if (typeof daVarLookupRev[key] != "undefined"){
            data[atob(daVarLookupRev[key])] = val;
          }
          else{
            data[atob(key)] = val;
          }
        }
        return JSON.stringify(data);
      }
      var daMessageLog = JSON.parse(atob(""" + json.dumps(safeid(json.dumps(docassemble.base.functions.get_message_log()))) + """));
      function daPreloadImage(url){
        var img = new Image();
        img.src = url;
      }
      daPreloadImage('""" + str(url_for('static', filename='app/chat.ico', v=da_version)) + """');
      function daShowHelpTab(){
          $('#dahelptoggle').tab('show');
      }
      function addCsrfHeader(xhr, settings){
        if (daJsEmbed && !/^(GET|HEAD|OPTIONS|TRACE)$/i.test(settings.type)){
          xhr.setRequestHeader("X-CSRFToken", daCsrf);
        }
      }
      function flash(message, priority, clear){
        if (priority == null){
          priority = 'info'
        }
        if (!$("#daflash").length){
          $(daTargetDiv).append(daSprintf(daNotificationContainer, ""));
        }
        if (clear){
          $("#daflash").empty();
        }
        if (message != null){
          $("#daflash").append(daSprintf(daNotificationMessage, priority, message));
          if (priority == 'success'){
            setTimeout(function(){
              $("#daflash .alert-success").hide(300, function(){
                $(this).remove();
              });
            }, 3000);
          }
        }
      }
      var da_flash = flash;
      function url_action(action, args){
        if (args == null){
          args = {};
        }
        data = {action: action, arguments: args};
        var url;
        if (daJsEmbed){
          url = daPostURL + "&action=" + encodeURIComponent(btoa(JSON_stringify(data)))
        }
        else{
          if (daLocationBar.indexOf('?') !== -1){
            url = daLocationBar + "&action=" + encodeURIComponent(btoa(JSON_stringify(data)))
          }
          else {
            url = daLocationBar + "?action=" + encodeURIComponent(btoa(JSON_stringify(data)))
          }
        }
        return url;
      }
      var da_url_action = url_action;
      function action_call(action, args, callback){
        if (args == null){
            args = {};
        }
        if (callback == null){
            callback = function(){};
        }
        var data = {action: action, arguments: args};
        var url;
        if (daJsEmbed){
          url = daPostURL + "&action=" + encodeURIComponent(btoa(JSON_stringify(data)))
        }
        else{
          url = daInterviewUrl + "&action=" + encodeURIComponent(btoa(JSON_stringify(data)))
        }
        return $.ajax({
          type: "GET",
          url: url,
          success: callback,
          beforeSend: addCsrfHeader,
          xhrFields: {
            withCredentials: true
          },
          error: function(xhr, status, error){
            setTimeout(function(){
              daProcessAjaxError(xhr, status, error);
            }, 0);
          }
        });
      }
      var da_action_call = action_call;
      var url_action_call = action_call;
      function action_perform(action, args){
        if (args == null){
            args = {};
        }
        var data = {action: action, arguments: args};
        daSpinnerTimeout = setTimeout(daShowSpinner, 1000);
        return $.ajax({
          type: "POST",
          url: daInterviewUrl,
          beforeSend: addCsrfHeader,
          xhrFields: {
            withCredentials: true
          },
          data: $.param({_action: btoa(JSON_stringify(data)), csrf_token: daCsrf, ajax: 1}),
          success: function(data){
            setTimeout(function(){
              daProcessAjax(data, $("#daform"), 1);
            }, 0);
          },
          error: function(xhr, status, error){
            setTimeout(function(){
              daProcessAjaxError(xhr, status, error);
            }, 0);
          },
          dataType: 'json'
        });
      }
      var da_action_perform = action_perform;
      var url_action_perform = action_perform;
      function action_perform_with_next(action, args, next_data){
        //console.log("action_perform_with_next: " + action + " | " + next_data)
        if (args == null){
            args = {};
        }
        var data = {action: action, arguments: args};
        daSpinnerTimeout = setTimeout(daShowSpinner, 1000);
        return $.ajax({
          type: "POST",
          url: daInterviewUrl,
          beforeSend: addCsrfHeader,
          xhrFields: {
            withCredentials: true
          },
          data: $.param({_action: btoa(JSON_stringify(data)), _next_action_to_set: btoa(JSON_stringify(next_data)), csrf_token: daCsrf, ajax: 1}),
          success: function(data){
            setTimeout(function(){
              daProcessAjax(data, $("#daform"), 1);
            }, 0);
          },
          error: function(xhr, status, error){
            setTimeout(function(){
              daProcessAjaxError(xhr, status, error);
            }, 0);
          },
          dataType: 'json'
        });
      }
      var da_action_perform_with_next = action_perform_with_next;
      var url_action_perform_with_next = action_perform_with_next;
      function get_interview_variables(callback){
        if (callback == null){
          callback = function(){};
        }
        return $.ajax({
          type: "GET",
          url: """ + '"' + url_for('get_variables', i=yaml_filename) + '"' + """,
          success: callback,
          beforeSend: addCsrfHeader,
          xhrFields: {
            withCredentials: true
          },
          error: function(xhr, status, error){
            setTimeout(function(){
              daProcessAjaxError(xhr, status, error);
            }, 0);
          }
        });
      }
      var da_get_interview_variables = get_interview_variables;
      function daInformAbout(subject, chatMessage){
        if (subject in daInformed || (subject != 'chatmessage' && !daIsUser)){
          return;
        }
        if (daShowingHelp && subject != 'chatmessage'){
          daInformed[subject] = 1;
          daInformedChanged = true;
          return;
        }
        if (daShowingHelp && subject == 'chatmessage'){
          return;
        }
        var target;
        var message;
        var waitPeriod = 3000;
        if (subject == 'chat'){
          target = "#daChatAvailable a";
          message = """ + json.dumps(word("Get help through live chat by clicking here.")) + """;
        }
        else if (subject == 'chatmessage'){
          target = "#daChatAvailable a";
          //message = """ + json.dumps(word("A chat message has arrived.")) + """;
          message = chatMessage;
        }
        else if (subject == 'phone'){
          target = "#daPhoneAvailable a";
          message = """ + json.dumps(word("Click here to get help over the phone.")) + """;
        }
        else{
          return;
        }
        if (subject != 'chatmessage'){
          daInformed[subject] = 1;
          daInformedChanged = true;
        }
        if (subject == 'chatmessage'){
          $(target).popover({"content": message, "placement": "bottom", "trigger": "manual", "container": "body", "title": """ + json.dumps(word("New chat message")) + """});
        }
        else {
          $(target).popover({"content": message, "placement": "bottom", "trigger": "manual", "container": "body", "title": """ + json.dumps(word("Live chat is available")) + """});
        }
        $(target).popover('show');
        setTimeout(function(){
          $(target).popover('dispose');
          $(target).removeAttr('title');
        }, waitPeriod);
      }
      // function daCloseSocket(){
      //   if (typeof daSocket !== 'undefined' && daSocket.connected){
      //     //daSocket.emit('terminate');
      //     //io.unwatch();
      //   }
      // }
      function daPublishMessage(data){
        var newDiv = document.createElement('li');
        $(newDiv).addClass("list-group-item");
        if (data.is_self){
          $(newDiv).addClass("list-group-item-primary dalistright");
        }
        else{
          $(newDiv).addClass("list-group-item-secondary dalistleft");
        }
        //var newSpan = document.createElement('span');
        //$(newSpan).html(data.message);
        //$(newSpan).appendTo($(newDiv));
        //var newName = document.createElement('span');
        //$(newName).html(userNameString(data));
        //$(newName).appendTo($(newDiv));
        $(newDiv).html(data.message);
        $("#daCorrespondence").append(newDiv);
      }
      function daScrollChat(){
        var chatScroller = $("#daCorrespondence");
        if (chatScroller.length){
          var height = chatScroller[0].scrollHeight;
          //console.log("Slow scrolling to " + height);
          if (height == 0){
            daNotYetScrolled = true;
            return;
          }
          chatScroller.animate({scrollTop: height}, 800);
        }
        else{
          console.log("daScrollChat: error");
        }
      }
      function daScrollChatFast(){
        var chatScroller = $("#daCorrespondence");
        if (chatScroller.length){
          var height = chatScroller[0].scrollHeight;
          if (height == 0){
            daNotYetScrolled = true;
            return;
          }
          //console.log("Scrolling to " + height + " where there are " + chatScroller[0].childElementCount + " children");
          chatScroller.scrollTop(height);
        }
        else{
          console.log("daScrollChatFast: error");
        }
      }
      function daSender(){
        //console.log("daSender");
        if ($("#daMessage").val().length){
          daSocket.emit('chatmessage', {data: $("#daMessage").val(), i: daYamlFilename});
          $("#daMessage").val("");
          $("#daMessage").focus();
        }
        return false;
      }
      function daShowControl(mode){
        //console.log("You are now being controlled");
        if ($("body").hasClass("dacontrolled")){
          return;
        }
        $('input[type="submit"], button[type="submit"]').prop("disabled", true);
        $("body").addClass("dacontrolled");
        var newDiv = document.createElement('div');
        $(newDiv).addClass("datop-alert col-xs-10 col-sm-7 col-md-6 col-lg-5 dacol-centered");
        $(newDiv).html(""" + json.dumps(word("Your screen is being controlled by an operator.")) + """)
        $(newDiv).attr('id', "dacontrolAlert");
        $(newDiv).css("display", "none");
        $(newDiv).appendTo($(daTargetDiv));
        if (mode == 'animated'){
          $(newDiv).slideDown();
        }
        else{
          $(newDiv).show();
        }
      }
      function daHideControl(){
        //console.log("You are no longer being controlled");
        if (! $("body").hasClass("dacontrolled")){
          return;
        }
        $('input[type="submit"], button[type="submit"]').prop("disabled", false);
        $("body").removeClass("dacontrolled");
        $("#dacontrolAlert").html(""" + json.dumps(word("The operator is no longer controlling your screen.")) + """);
        setTimeout(function(){
          $("#dacontrolAlert").slideUp(300, function(){
            $("#dacontrolAlert").remove();
          });
        }, 2000);
      }
      function daInitializeSocket(){
        if (daSocket != null){
            if (daSocket.connected){
                //console.log("Calling connectagain");
                if (daChatStatus == 'ready'){
                  daSocket.emit('connectagain', {i: daYamlFilename});
                }
                if (daBeingControlled){
                    daShowControl('animated');
                    daSocket.emit('start_being_controlled', {i: daYamlFilename});
                }
            }
            else{
                //console.log('daInitializeSocket: daSocket.connect()');
                daSocket.connect();
            }
            return;
        }
        if (location.protocol === 'http:' || document.location.protocol === 'http:'){
            daSocket = io.connect('http://' + document.domain + '/wsinterview', {path: '""" + ROOT + """ws/socket.io', query: "i=" + daYamlFilename});
        }
        if (location.protocol === 'https:' || document.location.protocol === 'https:'){
            daSocket = io.connect('https://' + document.domain + '/wsinterview', {path: '""" + ROOT + """ws/socket.io', query: "i=" + daYamlFilename});
        }
        //console.log("daInitializeSocket: socket is " + daSocket);
        if (daSocket != null){
            daSocket.on('connect', function() {
                if (daSocket == null){
                    console.log("Error: socket is null");
                    return;
                }
                //console.log("Connected socket with sid " + daSocket.id);
                if (daChatStatus == 'ready'){
                    daChatStatus = 'on';
                    daDisplayChat();
                    daPushChanges();
                    //daTurnOnChat();
                    //console.log("Emitting chat_log from on connect");
                    daSocket.emit('chat_log', {i: daYamlFilename});
                }
                if (daBeingControlled){
                    daShowControl('animated')
                    daSocket.emit('start_being_controlled', {i: daYamlFilename});
                }
            });
            daSocket.on('chat_log', function(arg) {
                //console.log("Got chat_log");
                $("#daCorrespondence").html('');
                daChatHistory = [];
                var messages = arg.data;
                for (var i = 0; i < messages.length; ++i){
                    daChatHistory.push(messages[i]);
                    daPublishMessage(messages[i]);
                }
                daScrollChatFast();
            });
            daSocket.on('chatready', function(data) {
                //var key = 'da:session:uid:' + data.uid + ':i:' + data.i + ':userid:' + data.userid
                //console.log('chatready');
            });
            daSocket.on('terminate', function() {
                //console.log("interview: terminating socket");
                daSocket.disconnect();
            });
            daSocket.on('controllerstart', function(){
              daBeingControlled = true;
              daShowControl('animated');
            });
            daSocket.on('controllerexit', function(){
              daBeingControlled = false;
              //console.log("Hiding control 2");
              daHideControl();
              if (daChatStatus != 'on'){
                if (daSocket != null && daSocket.connected){
                  //console.log('Terminating interview socket because control over');
                  daSocket.emit('terminate');
                }
              }
            });
            daSocket.on('disconnect', function() {
                //console.log("Manual disconnect");
                //daSocket.emit('manual_disconnect', {i: daYamlFilename});
                //console.log("Disconnected socket");
                //daSocket = null;
            });
            daSocket.on('reconnected', function() {
                //console.log("Reconnected");
                daChatStatus = 'on';
                daDisplayChat();
                daPushChanges();
                daTurnOnChat();
                //console.log("Emitting chat_log from reconnected");
                daSocket.emit('chat_log', {i: daYamlFilename});
            });
            daSocket.on('mymessage', function(arg) {
                //console.log("Received " + arg.data);
                $("#daPushResult").html(arg.data);
            });
            daSocket.on('departure', function(arg) {
                //console.log("Departure " + arg.numpartners);
                if (arg.numpartners == 0){
                    daCloseChat();
                }
            });
            daSocket.on('chatmessage', function(arg) {
                //console.log("Received chat message " + arg.data);
                daChatHistory.push(arg.data);
                daPublishMessage(arg.data);
                daScrollChat();
                daInformAbout('chatmessage', arg.data.message);
            });
            daSocket.on('newpage', function(incoming) {
                //console.log("newpage received");
                var data = incoming.obj;
                daProcessAjax(data, $("#daform"), 1);
            });
            daSocket.on('controllerchanges', function(data) {
                //console.log("controllerchanges: " + data.parameters);
                var valArray = Object();
                var values = JSON.parse(data.parameters);
                for (var i = 0; i < values.length; i++) {
                    valArray[values[i].name] = values[i].value;
                }
                //console.log("valArray is " + JSON.stringify(valArray));
                $("#daform").each(function(){
                    $(this).find(':input').each(function(){
                        var type = $(this).attr('type');
                        var id = $(this).attr('id');
                        var name = $(this).attr('name');
                        if (type == 'checkbox'){
                            if (name in valArray){
                                if (valArray[name] == 'True'){
                                    if ($(this).prop('checked') != true){
                                        $(this).prop('checked', true);
                                        $(this).trigger('change');
                                    }
                                }
                                else{
                                    if ($(this).prop('checked') != false){
                                        $(this).prop('checked', false);
                                        $(this).trigger('change');
                                    }
                                }
                            }
                            else{
                                if ($(this).prop('checked') != false){
                                    $(this).prop('checked', false);
                                    $(this).trigger('change');
                                }
                            }
                        }
                        else if (type == 'radio'){
                            if (name in valArray){
                                if (valArray[name] == $(this).val()){
                                    if ($(this).prop('checked') != true){
                                        $(this).prop('checked', true);
                                        $(this).trigger('change');
                                    }
                                }
                                else{
                                    if ($(this).prop('checked') != false){
                                        $(this).prop('checked', false);
                                        $(this).trigger('change');
                                    }
                                }
                            }
                        }
                        else if ($(this).data().hasOwnProperty('sliderMax')){
                            $(this).slider('setValue', parseInt(valArray[name]));
                        }
                        else{
                            if (name in valArray){
                                $(this).val(valArray[name]);
                            }
                        }
                    });
                });
                if (data.clicked){
                    //console.log("Need to click " + data.clicked);
                    $(data.clicked).prop("disabled", false);
                    $(data.clicked).addClass("da-click-selected");
                    if ($(data.clicked).prop("tagName") == 'A' && typeof $(data.clicked).attr('href') != 'undefined' && ($(data.clicked).attr('href').indexOf('javascript') == 0 || $(data.clicked).attr('href').indexOf('#') == 0)){
                      setTimeout(function(){
                        $(data.clicked).removeClass("da-click-selected");
                      }, 2200);
                    }
                    setTimeout(function(){
                      //console.log("Clicking it now");
                      $(data.clicked).click();
                      //console.log("Clicked it.");
                    }, 200);
                }
            });
        }
      }
      var daCheckinSeconds = """ + str(the_checkin_interval) + """;
      var daCheckinInterval = null;
      var daReloader = null;
      var daDisable = null;
      var daChatRoles = """ + json.dumps(user_dict['_internal']['livehelp']['roles']) + """;
      var daChatPartnerRoles = """ + json.dumps(user_dict['_internal']['livehelp']['partner_roles']) + """;
      function daUnfakeHtmlResponse(text){
        text = text.substr(text.indexOf('ABCDABOUNDARYSTARTABC') + 21);
        text = text.substr(0, text.indexOf('ABCDABOUNDARYENDABC')).replace(/\s/g, '');
        text = atob(text);
        return text;
      }
      function daInjectTrim(handler){
        return function (element, event) {
          if (element.tagName === "TEXTAREA" || (element.tagName === "INPUT" && element.type !== "password" && element.type !== "date" && element.type !== "datetime" && element.type !== "file")) {
            setTimeout(function(){
              element.value = $.trim(element.value);
            }, 10);
          }
          return handler.call(this, element, event);
        };
      }
      function daInvalidHandler(form, validator){
        var errors = validator.numberOfInvalids();
        var scrollTarget = null;
        if (errors && $(validator.errorList[0].element).parents('.da-form-group').length > 0) {
          if (daJsEmbed){
            scrollTarget = $(validator.errorList[0].element).parents('.da-form-group').first().position().top - 60;
          }
          else{
            scrollTarget = $(validator.errorList[0].element).parents('.da-form-group').first().offset().top - 60;
          }
        }
        if (scrollTarget != null){
          if (daJsEmbed){
            $(daTargetDiv).animate({
              scrollTop: scrollTarget
            }, 1000);
          }
          else{
            $("html, body").animate({
              scrollTop: scrollTarget
            }, 1000);
          }
        }
      }
      function daValidationHandler(form){
        //form.submit();
        //console.log("daValidationHandler");
        var visibleElements = [];
        var seen = Object();
        $(form).find("input, select, textarea").filter(":not(:disabled)").each(function(){
          //console.log("Considering an element");
          if ($(this).attr('name') && $(this).attr('type') != "hidden" && (($(this).hasClass('da-active-invisible') && $(this).parent().is(":visible")) || $(this).is(":visible"))){
            var theName = $(this).attr('name');
            //console.log("Including an element " + theName);
            if (!seen.hasOwnProperty(theName)){
              visibleElements.push(theName);
              seen[theName] = 1;
            }
          }
        });
        $(form).find("input[name='_visible']").val(btoa(JSON_stringify(visibleElements)));
        $(form).each(function(){
          $(this).find(':input').off('change', daPushChanges);
        });
        $("meta[name=viewport]").attr('content', "width=device-width, minimum-scale=1.0, maximum-scale=1.0, initial-scale=1.0");
        if (daCheckinInterval != null){
          clearInterval(daCheckinInterval);
        }
        daDisable = setTimeout(function(){
          $(form).find('input[type="submit"]').prop("disabled", true);
          $(form).find('button[type="submit"]').prop("disabled", true);
        }, 1);
        if (daWhichButton != null){
          $(".da-field-buttons .btn-da").each(function(){
            if (this != daWhichButton){
              $(this).removeClass(""" + '"' + app.config['BUTTON_STYLE'] + """primary """ + app.config['BUTTON_STYLE'] + """info """ + app.config['BUTTON_STYLE'] + """warning """ + app.config['BUTTON_STYLE'] + """danger """ + app.config['BUTTON_STYLE'] + """secondary");
              $(this).addClass(""" + '"' + app.config['BUTTON_STYLE'] + """light");
            }
          });
          if ($(daWhichButton).hasClass(""" + '"' + app.config['BUTTON_STYLE'] + """success")){
            $(daWhichButton).removeClass(""" + '"' + app.config['BUTTON_STYLE'] + """success");
            $(daWhichButton).addClass(""" + '"' + app.config['BUTTON_STYLE'] + """primary");
          }
          else{
            $(daWhichButton).removeClass(""" + '"' + app.config['BUTTON_STYLE'] + """primary """ + app.config['BUTTON_STYLE'] + """info """ + app.config['BUTTON_STYLE'] + """warning """ + app.config['BUTTON_STYLE'] + """danger """ + app.config['BUTTON_STYLE'] + """success """ + app.config['BUTTON_STYLE'] + """light");
            $(daWhichButton).addClass(""" + '"' + app.config['BUTTON_STYLE'] + """secondary");
          }
        }
        var tableOrder = {};
        var tableOrderChanges = {};
        $("a.datableup").each(function(){
          var tableName = $(this).data('tablename');
          if (!tableOrder.hasOwnProperty(tableName)){
            tableOrder[tableName] = [];
          }
          tableOrder[tableName].push(parseInt($(this).data('tableitem')));
        });
        var tableChanged = false;
        for (var tableName in tableOrder){
          if (tableOrder.hasOwnProperty(tableName)){
            var n = tableOrder[tableName].length;
            for (var i = 0; i < n; ++i){
              if (i != tableOrder[tableName][i]){
                tableChanged = true;
                if (!tableOrderChanges.hasOwnProperty(tableName)){
                  tableOrderChanges[tableName] = [];
                }
                tableOrderChanges[tableName].push([tableOrder[tableName][i], i])
              }
            }
          }
        }
        if (tableChanged){
          $('<input>').attr({
            type: 'hidden',
            name: '_order_changes',
            value: JSON.stringify(tableOrderChanges)
          }).appendTo($(form));
        }
        var collectToDelete = [];
        $(".dacollectunremove:visible").each(function(){
          collectToDelete.push(parseInt($(this).parent().parent().data('collectnum')));
        });
        var lastOk = parseInt($(".dacollectremove:visible, .dacollectremoveexisting:visible").last().parent().parent().data('collectnum'));
        $(".dacollectremove, .dacollectremoveexisting").each(function(){
          if (parseInt($(this).parent().parent().data('collectnum')) > lastOk){
            collectToDelete.push(parseInt($(this).parent().parent().data('collectnum')));
          }
        });
        if (collectToDelete.length > 0){
          $('<input>').attr({
            type: 'hidden',
            name: '_collect_delete',
            value: JSON.stringify(collectToDelete)
          }).appendTo($(form));
        }
       $("select.damultiselect:not(:disabled)").each(function(){
          var showifParents = $(this).parents(".dajsshowif,.dashowif");
          if (showifParents.length == 0 || $(showifParents[0]).data("isVisible") == '1'){
            $(this).find('option').each(function(){
              $('<input>').attr({
                type: 'hidden',
                name: $(this).val(),
                value: $(this).prop('selected') ? 'True' : 'False'
              }).appendTo($(form));
            });
          }
          $(this).prop('disabled', true);
        });
        daWhichButton = null;
        if (daSubmitter != null){
          $('<input>').attr({
            type: 'hidden',
            name: daSubmitter.name,
            value: daSubmitter.value
          }).appendTo($(form));
        }
        if (daInformedChanged){
          $("<input>").attr({
            type: 'hidden',
            name: 'informed',
            value: Object.keys(daInformed).join(',')
          }).appendTo($(form));
        }
        $('<input>').attr({
          type: 'hidden',
          name: 'ajax',
          value: '1'
        }).appendTo($(form));
        daSpinnerTimeout = setTimeout(daShowSpinner, 1000);
        var do_iframe_upload = false;
        inline_succeeded = false;
        if ($('input[name="_files"]').length){
          var filesToRead = 0;
          var filesRead = 0;
          var newFileList = Array();
          var nullFileList = Array();
          var fileArray = {keys: Array(), values: Object()};
          var file_list = JSON.parse(atob($('input[name="_files"]').val()));
          var inline_file_list = Array();
          var namesWithImages = Object();
          for (var i = 0; i < file_list.length; i++){
            var the_file_input = $('#' + file_list[i].replace(/(:|\.|\[|\]|,|=|\/|\")/g, '\\\\$1'))[0];
            var the_max_size = $(the_file_input).data('maximagesize');
            var the_image_type = $(the_file_input).data('imagetype');
            var hasImages = false;
            if (typeof the_max_size != 'undefined' || typeof the_image_type != 'undefined'){
              for (var j = 0; j < the_file_input.files.length; j++){
                var the_file = the_file_input.files[j];
                if (the_file.type.match(/image.*/)){
                  hasImages = true;
                }
              }
            }
            if (hasImages || (daJsEmbed && the_file_input.files.length > 0)){
              for (var j = 0; j < the_file_input.files.length; j++){
                var the_file = the_file_input.files[j];
                filesToRead++;
              }
              inline_file_list.push(file_list[i]);
            }
            else if (the_file_input.files.length > 0){
              newFileList.push(file_list[i]);
            }
            else{
              nullFileList.push(file_list[i]);
            }
            namesWithImages[file_list[i]] = hasImages;
          }
          if (inline_file_list.length > 0){
            var originalFileList = atob($('input[name="_files"]').val())
            if (newFileList.length == 0 && nullFileList.length == 0){
              $('input[name="_files"]').remove();
            }
            else{
              $('input[name="_files"]').val(btoa(JSON_stringify(newFileList.concat(nullFileList))));
            }
            for (var i = 0; i < inline_file_list.length; i++){
              fileArray.keys.push(inline_file_list[i])
              fileArray.values[inline_file_list[i]] = Array()
              var fileInfoList = fileArray.values[inline_file_list[i]];
              var file_input = $('#' + inline_file_list[i].replace(/(:|\.|\[|\]|,|=|\/|\")/g, '\\\\$1'))[0];
              var max_size;
              var image_type;
              var image_mime_type;
              var this_has_images = false;
              if (namesWithImages[inline_file_list[i]]){
                this_has_images = true;
                max_size = parseInt($(file_input).data('maximagesize'));
                image_type = $(file_input).data('imagetype');
                image_mime_type = null;
                if (image_type){
                  if (image_type == 'png'){
                    image_mime_type = 'image/png';
                  }
                  else if (image_type == 'bmp'){
                    image_mime_type = 'image/bmp';
                  }
                  else {
                    image_mime_type = 'image/jpeg';
                    image_type = 'jpg';
                  }
                }
              }
              for (var j = 0; j < file_input.files.length; j++){
                var a_file = file_input.files[j];
                var tempFunc = function(the_file, max_size, has_images){
                  var reader = new FileReader();
                  var thisFileInfo = {name: the_file.name, size: the_file.size, type: the_file.type};
                  fileInfoList.push(thisFileInfo);
                  reader.onload = function(readerEvent){
                    if (has_images && the_file.type.match(/image.*/) && !(the_file.type.indexOf('image/svg') == 0)){
                      var convertedName = the_file.name;
                      var convertedType = the_file.type;
                      if (image_type){
                        var pos = the_file.name.lastIndexOf(".");
                        convertedName = the_file.name.substr(0, pos < 0 ? the_file.name.length : pos) + "." + image_type;
                        convertedType = image_mime_type;
                        thisFileInfo.name = convertedName;
                        thisFileInfo.type = convertedType;
                      }
                      var image = new Image();
                      image.onload = function(imageEvent) {
                        var canvas = document.createElement('canvas'),
                          width = image.width,
                          height = image.height;
                        if (width > height) {
                          if (width > max_size) {
                              height *= max_size / width;
                              width = max_size;
                          }
                        }
                        else {
                          if (height > max_size) {
                            width *= max_size / height;
                            height = max_size;
                          }
                        }
                        canvas.width = width;
                        canvas.height = height;
                        canvas.getContext('2d').drawImage(image, 0, 0, width, height);
                        thisFileInfo['content'] = canvas.toDataURL(convertedType);
                        filesRead++;
                        if (filesRead >= filesToRead){
                          daResumeUploadSubmission(form, fileArray, inline_file_list, newFileList);
                        }
                      };
                      image.src = reader.result;
                    }
                    else{
                      thisFileInfo['content'] = reader.result;
                      filesRead++;
                      if (filesRead >= filesToRead){
                        daResumeUploadSubmission(form, fileArray, inline_file_list, newFileList);
                      }
                    }
                  };
                  reader.readAsDataURL(the_file);
                };
                tempFunc(a_file, max_size, this_has_images);
                inline_succeeded = true;
              }
            }
          }
          if (newFileList.length == 0){
            //$('input[name="_files"]').remove();
          }
          else{
            do_iframe_upload = true;
          }
        }
        if (inline_succeeded){
          return(false);
        }
        if (do_iframe_upload){
          $("#dauploadiframe").remove();
          var iframe = $('<iframe name="dauploadiframe" id="dauploadiframe" style="display: none"><\/iframe>');
          $(daTargetDiv).append(iframe);
          $(form).attr("target", "dauploadiframe");
          iframe.bind('load', function(){
            setTimeout(function(){
              try {
                daProcessAjax($.parseJSON(daUnfakeHtmlResponse($("#dauploadiframe").contents().text())), form, 1);
              }
              catch (e){
                try {
                  daProcessAjax($.parseJSON($("#dauploadiframe").contents().text()), form, 1);
                }
                catch (f){
                  daShowErrorScreen(document.getElementById('dauploadiframe').contentWindow.document.body.innerHTML, f);
                }
              }
            }, 0);
          });
          form.submit();
        }
        else{
          $.ajax({
            type: "POST",
            url: daInterviewUrl,
            data: $(form).serialize(),
            beforeSend: addCsrfHeader,
            xhrFields: {
              withCredentials: true
            },
            success: function(data){
              setTimeout(function(){
                daProcessAjax(data, form, 1);
              }, 0);
            },
            error: function(xhr, status, error){
              setTimeout(function(){
                daProcessAjaxError(xhr, status, error);
              }, 0);
            }
          });
        }
        return(false);
      }
      function daSignatureSubmit(event){
        $(this).find("input[name='ajax']").val(1);
        $.ajax({
          type: "POST",
          url: daInterviewUrl,
          data: $(this).serialize(),
          beforeSend: addCsrfHeader,
          xhrFields: {
            withCredentials: true
          },
          success: function(data){
            setTimeout(function(){
              daProcessAjax(data, $(this), 1);
            }, 0);
          },
          error: function(xhr, status, error){
            setTimeout(function(){
              daProcessAjaxError(xhr, status, error);
            }, 0);
          }
        });
        event.preventDefault();
        event.stopPropagation();
        return(false);
      }
      function JSON_stringify(s){
         var json = JSON.stringify(s);
         return json.replace(/[\\u007f-\\uffff]/g,
            function(c) {
              return '\\\\u'+('0000'+c.charCodeAt(0).toString(16)).slice(-4);
            }
         );
      }
      function daResumeUploadSubmission(form, fileArray, inline_file_list, newFileList){
        $('<input>').attr({
          type: 'hidden',
          name: '_files_inline',
          value: btoa(JSON_stringify(fileArray))
        }).appendTo($(form));
        for (var i = 0; i < inline_file_list.length; ++i){
          document.getElementById(inline_file_list[i]).disabled = true;
        }
        if (newFileList.length > 0){
          $("#dauploadiframe").remove();
          var iframe = $('<iframe name="dauploadiframe" id="dauploadiframe" style="display: none"><\/iframe>');
          $(daTargetDiv).append(iframe);
          $(form).attr("target", "dauploadiframe");
          iframe.bind('load', function(){
            setTimeout(function(){
              daProcessAjax($.parseJSON($("#dauploadiframe").contents().text()), form, 1);
            }, 0);
          });
          form.submit();
        }
        else{
          $.ajax({
            type: "POST",
            url: daInterviewUrl,
            data: $(form).serialize(),
            beforeSend: addCsrfHeader,
            xhrFields: {
              withCredentials: true
            },
            success: function(data){
              setTimeout(function(){
                daProcessAjax(data, form, 1);
              }, 0);
            },
            error: function(xhr, status, error){
              setTimeout(function(){
                daProcessAjaxError(xhr, status, error);
              }, 0);
            }
          });
        }
      }
      function daPushChanges(){
        //console.log("daPushChanges");
        if (daCheckinSeconds == 0 || daShowIfInProcess){
          return true;
        }
        if (daCheckinInterval != null){
          clearInterval(daCheckinInterval);
        }
        daCheckin();
        daCheckinInterval = setInterval(daCheckin, daCheckinSeconds);
        return true;
      }
      function daProcessAjaxError(xhr, status, error){
        if (xhr.responseType == undefined || xhr.responseType == '' || xhr.responseType == 'text'){
          var theHtml = xhr.responseText;
          if (theHtml == undefined){
            $(daTargetDiv).html("error");
          }
          else{
            theHtml = theHtml.replace(/<script[^>]*>[^<]*<\/script>/g, '');
            $(daTargetDiv).html(theHtml);
          }
          if (daJsEmbed){
            $(daTargetDiv)[0].scrollTo(0, 1);
          }
          else{
            window.scrollTo(0, 1);
          }
        }
        else {
          console.log("daProcessAjaxError: response was not text");
        }
      }
      function daAddScriptToHead(src){
        var head = document.getElementsByTagName("head")[0];
        var script = document.createElement("script");
        script.type = "text/javascript";
        script.src = src;
        script.async = true;
        script.defer = true;
        head.appendChild(script);
      }
      $(document).on('keydown', function(e){
        if (e.which == 13){
          if (daShowingHelp == 0){
            var tag = $( document.activeElement ).prop("tagName");
            if (tag != "INPUT" && tag != "TEXTAREA" && tag != "A" && tag != "LABEL" && tag != "BUTTON"){
              e.preventDefault();
              e.stopPropagation();
              if ($("#daform .da-field-buttons button").not('.danonsubmit').length == 1){
                $("#daform .da-field-buttons button").not('.danonsubmit').click();
              }
              return false;
            }
          }
          if ($(document.activeElement).hasClass("btn-file")){
            e.preventDefault();
            e.stopPropagation();
            $(document.activeElement).find('input').click();
            return false;
          }
        }
      });
      function daShowErrorScreen(data, error){
        console.log('daShowErrorScreen: ' + error);
        if ("activeElement" in document){
          document.activeElement.blur();
        }
        $(daTargetDiv).html(data);
      }
      function daProcessAjax(data, form, doScroll){
        daInformedChanged = false;
        if (daDisable != null){
          clearTimeout(daDisable);
        }
        daCsrf = data.csrf_token;
        if (data.question_data){
          daQuestionData = data.question_data;
        }
        if (data.action == 'body'){""" + forceFullScreen + """
          if ("activeElement" in document){
            document.activeElement.blur();
          }
          $(daTargetDiv).html(data.body);
          var bodyClasses = $(daTargetDiv).parent()[0].className.split(/\s+/);
          var n = bodyClasses.length;
          while (n--){
            if (bodyClasses[n] == 'dabody' || bodyClasses[n] == 'dasignature' || bodyClasses[n].indexOf('question-') == 0){
              $(daTargetDiv).parent().removeClass(bodyClasses[n]);
            }
          }
          $(daTargetDiv).parent().addClass(data.bodyclass);
          $("meta[name=viewport]").attr('content', "width=device-width, initial-scale=1");
          daDoAction = data.do_action;
          //daNextAction = data.next_action;
          daChatAvailable = data.livehelp.availability;
          daChatMode = data.livehelp.mode;
          daChatRoles = data.livehelp.roles;
          daChatPartnerRoles = data.livehelp.partner_roles;
          daSteps = data.steps;
          //console.log("daProcessAjax: pushing " + daSteps);
          if (!daJsEmbed && !daIframeEmbed){
            if (history.state != null && daSteps > history.state.steps){
              history.pushState({steps: daSteps}, data.browser_title + " - page " + daSteps, daLocationBar + """ + json.dumps(page_sep) + """ + daSteps);
            }
            else{
              history.replaceState({steps: daSteps}, "", daLocationBar + """ + json.dumps(page_sep) + """ + daSteps);
            }
          }
          daAllowGoingBack = data.allow_going_back;
          daQuestionID = data.id_dict;
          daMessageLog = data.message_log;
          daInitialize(doScroll);
          var tempDiv = document.createElement('div');
          tempDiv.innerHTML = data.extra_scripts;
          var scripts = tempDiv.getElementsByTagName('script');
          for (var i = 0; i < scripts.length; i++){
            //console.log("Found one script");
            if (scripts[i].src != ""){
              //console.log("Added script to head");
              daAddScriptToHead(scripts[i].src);
            }
            else{
              daGlobalEval(scripts[i].innerHTML);
            }
          }
          $(".da-group-has-error").each(function(){
            if ($(this).is(":visible")){
              if (daJsEmbed){
                var scrollToTarget = $(this).position().top - 60;
                setTimeout(function(){
                  $(daTargetDiv).animate({scrollTop: scrollToTarget}, 1000);
                }, 100);
              }
              else{
                var scrollToTarget = $(this).offset().top - 60;
                setTimeout(function(){
                  $(daTargetDiv).parent().parent().animate({scrollTop: scrollToTarget}, 1000);
                }, 100);
              }
              return false;
            }
          });
          for (var i = 0; i < data.extra_css.length; i++){
            $("head").append(data.extra_css[i]);
          }
          document.title = data.browser_title;
          if ($("html").attr("lang") != data.lang){
            $("html").attr("lang", data.lang);
          }
          if (daReloader != null){
            clearTimeout(daReloader);
          }
          if (data.reload_after != null && data.reload_after > 0){
            //daReloader = setTimeout(function(){location.reload();}, data.reload_after);
            daReloader = setTimeout(function(){daRefreshSubmit();}, data.reload_after);
          }
          daUpdateHeight();
        }
        else if (data.action == 'redirect'){
          if (daSpinnerTimeout != null){
            clearTimeout(daSpinnerTimeout);
            daSpinnerTimeout = null;
          }
          if (daShowingSpinner){
            daHideSpinner();
          }
          window.location = data.url;
        }
        else if (data.action == 'refresh'){
          daRefreshSubmit();
        }
        else if (data.action == 'reload'){
          location.reload(true);
        }
        else if (data.action == 'resubmit'){
          $("input[name='ajax']").remove();
          if (daSubmitter != null){
            var input = $("<input>")
              .attr("type", "hidden")
              .attr("name", daSubmitter.name).val(daSubmitter.value);
            $(form).append($(input));
          }
          form.submit();
        }
      }
      function daEmbeddedJs(e){
        //console.log("using embedded js");
        var data = decodeURIComponent($(this).data('js'));
        daGlobalEval(data);
        e.preventDefault();
        return false;
      }
      function daEmbeddedAction(e){
        if ($(this).hasClass("daremovebutton")){
          if (confirm(""" + json.dumps(word("Are you sure you want to delete this item?")) + """)){
            return true;
          }
          e.preventDefault();
          $(this).blur();
          return false;
        }
        var data = decodeURIComponent($(this).data('embaction'));
        $.ajax({
          type: "POST",
          url: daInterviewUrl,
          data: $.param({_action: data, csrf_token: daCsrf, ajax: 1}),
          beforeSend: addCsrfHeader,
          xhrFields: {
            withCredentials: true
          },
          success: function(data){
            setTimeout(function(){
              daProcessAjax(data, $("#daform"), 1);
            }, 0);
          },
          error: function(xhr, status, error){
            setTimeout(function(){
              daProcessAjaxError(xhr, status, error);
            }, 0);
          },
          dataType: 'json'
        });
        daSpinnerTimeout = setTimeout(daShowSpinner, 1000);
        e.preventDefault();
        return false;
      }
      function daReviewAction(e){
        //action_perform_with_next($(this).data('action'), null, daNextAction);
        var info = $.parseJSON(atob($(this).data('action')));
        da_action_perform(info['action'], info['arguments']);
        e.preventDefault();
        return false;
      }
      function daRingChat(){
        daChatStatus = 'ringing';
        daPushChanges();
      }
      function daTurnOnChat(){
        //console.log("Publishing from daTurnOnChat");
        $("#daChatOnButton").addClass("dainvisible");
        $("#daChatBox").removeClass("dainvisible");
        $("#daCorrespondence").html('');
        for(var i = 0; i < daChatHistory.length; i++){
          daPublishMessage(daChatHistory[i]);
        }
        daScrollChatFast();
        $("#daMessage").prop('disabled', false);
        if (daShowingHelp){
          $("#daMessage").focus();
        }
      }
      function daCloseChat(){
        //console.log('daCloseChat');
        daChatStatus = 'hangup';
        daPushChanges();
        if (daSocket != null && daSocket.connected){
          daSocket.disconnect();
        }
      }
      // function daTurnOffChat(){
      //   $("#daChatOnButton").removeClass("dainvisible");
      //   $("#daChatBox").addClass("dainvisible");
      //   //daCloseSocket();
      //   $("#daMessage").prop('disabled', true);
      //   $("#daSend").unbind();
      //   //daStartCheckingIn();
      // }
      function daDisplayChat(){
        if (daChatStatus == 'off' || daChatStatus == 'observeonly'){
          $("#daChatBox").addClass("dainvisible");
          $("#daChatAvailable").addClass("dainvisible");
          $("#daChatOnButton").addClass("dainvisible");
        }
        else{
          if (daChatStatus == 'waiting'){
            if (daChatPartnersAvailable > 0){
              $("#daChatBox").removeClass("dainvisible");
            }
          }
          else {
            $("#daChatBox").removeClass("dainvisible");
          }
        }
        if (daChatStatus == 'waiting'){
          //console.log("I see waiting")
          if (daChatHistory.length > 0){
            $("#daChatAvailable a i").removeClass("da-chat-active");
            $("#daChatAvailable a i").addClass("da-chat-inactive");
            $("#daChatAvailable").removeClass("dainvisible");
          }
          else{
            $("#daChatAvailable a i").removeClass("da-chat-active");
            $("#daChatAvailable a i").removeClass("da-chat-inactive");
            $("#daChatAvailable").addClass("dainvisible");
          }
          $("#daChatOnButton").addClass("dainvisible");
          $("#daChatOffButton").addClass("dainvisible");
          $("#daMessage").prop('disabled', true);
          $("#daSend").prop('disabled', true);
        }
        if (daChatStatus == 'standby' || daChatStatus == 'ready'){
          //console.log("I see standby")
          $("#daChatAvailable").removeClass("dainvisible");
          $("#daChatAvailable a i").removeClass("da-chat-inactive");
          $("#daChatAvailable a i").addClass("da-chat-active");
          $("#daChatOnButton").removeClass("dainvisible");
          $("#daChatOffButton").addClass("dainvisible");
          $("#daMessage").prop('disabled', true);
          $("#daSend").prop('disabled', true);
          daInformAbout('chat');
        }
        if (daChatStatus == 'on'){
          $("#daChatAvailable").removeClass("dainvisible");
          $("#daChatAvailable a i").removeClass("da-chat-inactive");
          $("#daChatAvailable a i").addClass("da-chat-active");
          $("#daChatOnButton").addClass("dainvisible");
          $("#daChatOffButton").removeClass("dainvisible");
          $("#daMessage").prop('disabled', false);
          if (daShowingHelp){
            $("#daMessage").focus();
          }
          $("#daSend").prop('disabled', false);
          daInformAbout('chat');
        }
      }
      function daChatLogCallback(data){
        if (data.action && data.action == 'reload'){
          location.reload(true);
        }
        //console.log("daChatLogCallback: success is " + data.success);
        if (data.success){
          $("#daCorrespondence").html('');
          daChatHistory = [];
          var messages = data.messages;
          for (var i = 0; i < messages.length; ++i){
            daChatHistory.push(messages[i]);
            daPublishMessage(messages[i]);
          }
          daDisplayChat();
          daScrollChatFast();
        }
      }
      function daRefreshSubmit(){
        $.ajax({
          type: "POST",
          url: daInterviewUrl,
          data: 'csrf_token=' + daCsrf + '&ajax=1',
          beforeSend: addCsrfHeader,
          xhrFields: {
            withCredentials: true
          },
          success: function(data){
            setTimeout(function(){
              daProcessAjax(data, $("#daform"), 0);
            }, 0);
          },
          error: function(xhr, status, error){
            setTimeout(function(){
              daProcessAjaxError(xhr, status, error);
            }, 0);
          }
        });
      }
      function daResetCheckinCode(){
        daCheckinCode = Math.random();
      }
      function daCheckinCallback(data){
        if (data.action && data.action == 'reload'){
          location.reload(true);
        }
        daCheckingIn = 0;
        //console.log("daCheckinCallback: success is " + data.success);
        if (data.checkin_code != daCheckinCode){
          console.log("Ignoring checkincallback because code is wrong");
          return;
        }
        if (data.success){
          if (data.commands.length > 0){
            for (var i = 0; i < data.commands.length; ++i){
              var command = data.commands[i];
              if (command.extra == 'flash'){
                if (!$("#daflash").length){
                  $(daTargetDiv).append(daSprintf(daNotificationContainer, ""));
                }
                $("#daflash").append(daSprintf(daNotificationMessage, "info", command.value));
                //console.log("command is " + command.value);
              }
              else if (command.extra == 'refresh'){
                daRefreshSubmit();
              }
              else if (command.extra == 'javascript'){
                //console.log("I should eval" + command.value);
                daGlobalEval(command.value);
              }
              else if (command.extra == 'fields'){
                for (var key in command.value){
                  if (command.value.hasOwnProperty(key)){
                    daSetField(key, command.value[key]);
                  }
                }
              }
              else if (command.extra == 'backgroundresponse'){
                var assignments = Array();
                if (command.value.hasOwnProperty('target') && command.value.hasOwnProperty('content')){
                  assignments.push({target: command.value.target, content: command.value.content});
                }
                if (Array.isArray(command.value)){
                  for (i = 0; i < command.value.length; ++i){
                    var possible_assignment = command.value[i];
                    if (possible_assignment.hasOwnProperty('target') && possible_assignment.hasOwnProperty('content')){
                      assignments.push({target: possible_assignment.target, content: possible_assignment.content});
                    }
                  }
                }
                for (i = 0; i < assignments.length; ++i){
                  var assignment = assignments[i];
                  $('.datarget' + assignment.target.replace(/[^A-Za-z0-9\_]/g)).prop('innerHTML', assignment.content);
                }
                //console.log("Triggering daCheckIn");
                $(document).trigger('daCheckIn', [command.action, command.value]);
              }
            }
            // setTimeout(function(){
            //   $("#daflash .daalert-interlocutory").hide(300, function(){
            //     $(self).remove();
            //   });
            // }, 5000);
          }
          oldDaChatStatus = daChatStatus;
          //console.log("daCheckinCallback: from " + daChatStatus + " to " + data.chat_status);
          if (data.phone == null){
            $("#daPhoneMessage").addClass("dainvisible");
            $("#daPhoneMessage p").html('');
            $("#daPhoneAvailable").addClass("dainvisible");
            daPhoneAvailable = false;
          }
          else{
            $("#daPhoneMessage").removeClass("dainvisible");
            $("#daPhoneMessage p").html(data.phone);
            $("#daPhoneAvailable").removeClass("dainvisible");
            daPhoneAvailable = true;
            daInformAbout('phone');
          }
          var statusChanged;
          if (daChatStatus == data.chat_status){
            statusChanged = false;
          }
          else{
            statusChanged = true;
          }
          if (statusChanged){
            daChatStatus = data.chat_status;
            daDisplayChat();
            if (daChatStatus == 'ready'){
              //console.log("calling initialize socket because ready");
              daInitializeSocket();
            }
          }
          daChatPartnersAvailable = 0;
          if (daChatMode == 'peer' || daChatMode == 'peerhelp'){
            daChatPartnersAvailable += data.num_peers;
            if (data.num_peers == 1){
              $("#dapeerMessage").html('<span class="badge bg-info">' + data.num_peers + ' ' + """ + json.dumps(word("other user")) + """ + '<\/span>');
            }
            else{
              $("#dapeerMessage").html('<span class="badge bg-info">' + data.num_peers + ' ' + """ + json.dumps(word("other users")) + """ + '<\/span>');
            }
            $("#dapeerMessage").removeClass("dainvisible");
          }
          else{
            $("#dapeerMessage").addClass("dainvisible");
          }
          if (daChatMode == 'peerhelp' || daChatMode == 'help'){
            if (data.help_available == 1){
              $("#dapeerHelpMessage").html('<span class="badge bg-primary">' + data.help_available + ' ' + """ + json.dumps(word("operator")) + """ + '<\/span>');
            }
            else{
              $("#dapeerHelpMessage").html('<span class="badge bg-primary">' + data.help_available + ' ' + """ + json.dumps(word("operators")) + """ + '<\/span>');
            }
            $("#dapeerHelpMessage").removeClass("dainvisible");
          }
          else{
            $("#dapeerHelpMessage").addClass("dainvisible");
          }
          if (daBeingControlled){
            if (!data.observerControl){
              daBeingControlled = false;
              //console.log("Hiding control 1");
              daHideControl();
              if (daChatStatus != 'on'){
                if (daSocket != null && daSocket.connected){
                  //console.log('Terminating interview socket because control is over');
                  daSocket.emit('terminate');
                }
              }
            }
          }
          else{
            if (data.observerControl){
              daBeingControlled = true;
              daInitializeSocket();
            }
          }
        }
      }
      function daCheckoutCallback(data){
      }
      function daCheckin(){
        //console.log("daCheckin");
        daCheckingIn += 1;
        //if (daCheckingIn > 1 && !(daCheckingIn % 3)){
        if (daCheckingIn > 1){
          //console.log("daCheckin: request already pending, not re-sending");
          return;
        }
        var datastring;
        if ((daChatStatus != 'off') && $("#daform").length > 0 && !daBeingControlled){ // daChatStatus == 'waiting' || daChatStatus == 'standby' || daChatStatus == 'ringing' || daChatStatus == 'ready' || daChatStatus == 'on' || daChatStatus == 'observeonly'
          if (daDoAction != null){
            datastring = $.param({action: 'checkin', chatstatus: daChatStatus, chatmode: daChatMode, csrf_token: daCsrf, checkinCode: daCheckinCode, parameters: daFormAsJSON(), raw_parameters: JSON.stringify($("#daform").serializeArray()), do_action: daDoAction, ajax: '1'});
          }
          else{
            datastring = $.param({action: 'checkin', chatstatus: daChatStatus, chatmode: daChatMode, csrf_token: daCsrf, checkinCode: daCheckinCode, parameters: daFormAsJSON(), raw_parameters: JSON.stringify($("#daform").serializeArray()), ajax: '1'});
          }
        }
        else{
          if (daDoAction != null){
            datastring = $.param({action: 'checkin', chatstatus: daChatStatus, chatmode: daChatMode, csrf_token: daCsrf, checkinCode: daCheckinCode, do_action: daDoAction, parameters: daFormAsJSON(), ajax: '1'});
          }
          else{
            datastring = $.param({action: 'checkin', chatstatus: daChatStatus, chatmode: daChatMode, csrf_token: daCsrf, checkinCode: daCheckinCode, ajax: '1'});
          }
        }
        //console.log("Doing checkin with " + daChatStatus);
        $.ajax({
          type: 'POST',
          url: """ + "'" + url_for('checkin', i=yaml_filename) + "'" + """,
          beforeSend: addCsrfHeader,
          xhrFields: {
            withCredentials: true
          },
          data: datastring,
          success: daCheckinCallback,
          dataType: 'json'
        });
        return true;
      }
      function daCheckout(){
        $.ajax({
          type: 'POST',
          url: """ + "'" + url_for('checkout', i=yaml_filename) + "'" + """,
          beforeSend: addCsrfHeader,
          xhrFields: {
            withCredentials: true
          },
          data: 'csrf_token=' + daCsrf + '&ajax=1&action=checkout',
          success: daCheckoutCallback,
          dataType: 'json'
        });
        return true;
      }
      function daStopCheckingIn(){
        daCheckout();
        if (daCheckinInterval != null){
          clearInterval(daCheckinInterval);
        }
      }
      function daShowSpinner(){
        if ($("#daquestion").length > 0){
          $('<div id="daSpinner" class="da-spinner-container da-top-for-navbar"><div class="container"><div class="row"><div class="col text-center"><span class="da-spinner"><i class="fas fa-spinner fa-spin"><\/i><\/span><\/div><\/div><\/div><\/div>').appendTo(daTargetDiv);
        }
        else{
          var newSpan = document.createElement('span');
          var newI = document.createElement('i');
          $(newI).addClass("fas fa-spinner fa-spin");
          $(newI).appendTo(newSpan);
          $(newSpan).attr("id", "daSpinner");
          $(newSpan).addClass("da-sig-spinner da-top-for-navbar");
          $(newSpan).appendTo("#dasigtoppart");
        }
        daShowingSpinner = true;
      }
      function daHideSpinner(){
        $("#daSpinner").remove();
        daShowingSpinner = false;
        daSpinnerTimeout = null;
      }
      function daAdjustInputWidth(e){
        var contents = $(this).val();
        var leftBracket = new RegExp('<', 'g');
        var rightBracket = new RegExp('>', 'g');
        contents = contents.replace(/&/g,'&amp;').replace(leftBracket,'&lt;').replace(rightBracket,'&gt;').replace(/ /g, '&nbsp;');
        $('<span class="dainput-embedded" id="dawidth">').html( contents ).appendTo('#daquestion');
        $("#dawidth").css('min-width', $(this).css('min-width'));
        $("#dawidth").css('background-color', $(daTargetDiv).css('background-color'));
        $("#dawidth").css('color', $(daTargetDiv).css('background-color'));
        $(this).width($('#dawidth').width() + 16);
        setTimeout(function(){
          $("#dawidth").remove();
        }, 0);
      }
      function daShowNotifications(){
        var n = daMessageLog.length;
        for (var i = 0; i < n; i++){
          var message = daMessageLog[i];
          if (message.priority == 'console'){
            console.log(message.message);
          }
          else if (message.priority == 'javascript'){
            daGlobalEval(message.message);
          }
          else if (message.priority == 'success' || message.priority == 'warning' || message.priority == 'danger' || message.priority == 'secondary' || message.priority == 'info' || message.priority == 'secondary' || message.priority == 'dark' || message.priority == 'light' || message.priority == 'primary'){
            da_flash(message.message, message.priority);
          }
          else{
            da_flash(message.message, 'info');
          }
        }
      }
      function daIgnoreAllButTab(event){
        event = event || window.event;
        var code = event.keyCode;
        if (code != 9){
          if (code == 13){
            $(event.target).parents(".file-caption-main").find("input.dafile").click();
          }
          event.preventDefault();
          return false;
        }
      }
      function daDisableIfNotHidden(query, value){
        $(query).each(function(){
          var showIfParent = $(this).parents('.dashowif,.dajsshowif');
          if (!(showIfParent.length && ($(showIfParent[0]).data('isVisible') == '0' || !$(showIfParent[0]).is(":visible")))){
            if ($(this).hasClass('combobox')){
              if (value){
                daComboBoxes[$(this).attr('id')].disable();
              }
              else {
                daComboBoxes[$(this).attr('id')].enable();
              }
            }
            else {
              $(this).prop("disabled", value);
            }
          }
        });
      }
      function daShowIfCompare(theVal, showIfVal){
        if (typeof theVal == 'string' && theVal.match(/^-?\d+\.\d+$/)){
          theVal = parseFloat(theVal);
        }
        else if (typeof theVal == 'string' && theVal.match(/^-?\d+$/)){
          theVal = parseInt(theVal);
        }
        if (typeof showIfVal == 'string' && showIfVal.match(/^-?\d+\.\d+$/)){
          showIfVal = parseFloat(showIfVal);
        }
        else if (typeof showIfVal == 'string' && showIfVal.match(/^-?\d+$/)){
          showIfVal = parseInt(showIfVal);
        }
        if (typeof theVal == 'string' || typeof showIfVal == 'string'){
          if (String(showIfVal) == 'None' && String(theVal) == ''){
            return true;
          }
          return (String(theVal) == String(showIfVal));
        }
        return (theVal == showIfVal);
      }
      function rationalizeListCollect(){
        var finalNum = $(".dacollectextraheader").last().data('collectnum');
        var num = $(".dacollectextraheader:visible").last().data('collectnum');
        if (parseInt(num) < parseInt(finalNum)){
          if ($('div.dacollectextraheader[data-collectnum="' + num + '"]').find(".dacollectadd").hasClass('dainvisible')){
            $('div.dacollectextraheader[data-collectnum="' + (num + 1) + '"]').show('fast');
          }
        }
        var n = parseInt(finalNum);
        var firstNum = parseInt($(".dacollectextraheader").first().data('collectnum'));
        while (n-- > firstNum){
          if ($('div.dacollectextraheader[data-collectnum="' + (n + 1) + '"]:visible').length > 0){
            if (!$('div.dacollectextraheader[data-collectnum="' + (n + 1) + '"]').find(".dacollectadd").hasClass('dainvisible') && $('div.dacollectextraheader[data-collectnum="' + n + '"]').find(".dacollectremove").hasClass('dainvisible')){
              $('div.dacollectextraheader[data-collectnum="' + (n + 1) + '"]').hide();
            }
          }
        }
        var n = parseInt(finalNum);
        var seenAddAnother = false;
        while (n-- > firstNum){
          if ($('div.dacollectextraheader[data-collectnum="' + (n + 1) + '"]:visible').length > 0){
            if (!$('div.dacollectextraheader[data-collectnum="' + (n + 1) + '"]').find(".dacollectadd").hasClass('dainvisible')){
              seenAddAnother = true;
            }
            var current = $('div.dacollectextraheader[data-collectnum="' + n + '"]');
            if (seenAddAnother && !$(current).find(".dacollectadd").hasClass('dainvisible')){
              $(current).find(".dacollectadd").addClass('dainvisible');
              $(current).find(".dacollectunremove").removeClass('dainvisible');
            }
          }
        }
      }
      function daFetchAjax(elem, cb, doShow){
        var wordStart = $(elem).val();
        if (wordStart.length < parseInt(cb.$source.data('trig'))){
          if (cb.shown){
            cb.hide();
          }
          return;
        }
        if (daFetchAjaxTimeout != null && daFetchAjaxTimeoutRunning){
          daFetchAjaxTimeoutFetchAfter = true;
          return;
        }
        if (doShow){
          daFetchAjaxTimeout = setTimeout(function(){
            daFetchAjaxTimeoutRunning = false;
            if (daFetchAjaxTimeoutFetchAfter){
              daFetchAjax(elem, cb, doShow);
              daFetchAjaxTimeoutFetchAfter = false;
            }
          }, 2000);
          daFetchAjaxTimeoutRunning = true;
          daFetchAjaxTimeoutFetchAfter = false;
        }
        da_action_call(cb.$source.data('action'), {wordstart: wordStart}, function(data){
          wordStart = $(elem).val();
          if (typeof data == "object"){
            var upperWordStart = wordStart.toUpperCase()
            cb.$source.empty();
            var emptyItem = $("<option>");
            emptyItem.val("");
            emptyItem.text("");
            cb.$source.append(emptyItem);
            var notYetSelected = true;
            var selectedValue = null;
            if (Array.isArray(data)){
              for (var i = 0; i < data.length; ++i){
                if (Array.isArray(data[i])){
                  if (data[i].length >= 2){
                    var item = $("<option>");
                    if (notYetSelected && ((doShow && data[i][1].toString() == wordStart) || data[i][0].toString() == wordStart)){
                      item.prop('selected', true);
                      notYetSelected = false;
                      selectedValue = data[i][1]
                    }
                    item.text(data[i][1]);
                    item.val(data[i][0]);
                    cb.$source.append(item);
                  }
                  else if (data[i].length == 1){
                    var item = $("<option>");
                    if (notYetSelected && ((doShow && data[i][0].toString() == wordStart) || data[i][0].toString() == wordStart)){
                      item.prop('selected', true);
                      notYetSelected = false;
                      selectedValue = data[i][0]
                    }
                    item.text(data[i][0]);
                    item.val(data[i][0]);
                    cb.$source.append(item);
                  }
                }
                else if (typeof data[i] == "object"){
                  for (var key in data[i]){
                    if (data[i].hasOwnProperty(key)){
                      var item = $("<option>");
                      if (notYetSelected && ((doShow && key.toString() == wordStart) || key.toString() == wordStart)){
                        item.prop('selected', true);
                        notYetSelected = false;
                        selectedValue = data[i][key];
                      }
                      item.text(data[i][key]);
                      item.val(key);
                      cb.$source.append(item);
                    }
                  }
                }
                else{
                  var item = $("<option>");
                  if (notYetSelected && ((doShow && data[i].toString().toUpperCase() == upperWordStart) || data[i].toString() == wordStart)){
                    item.prop('selected', true);
                    notYetSelected = false;
                    selectedValue = data[i];
                  }
                  item.text(data[i]);
                  item.val(data[i]);
                  cb.$source.append(item);
                }
              }
            }
            else if (typeof data == "object"){
              var keyList = Array();
              for (var key in data){
                if (data.hasOwnProperty(key)){
                  keyList.push(key);
                }
              }
              keyList = keyList.sort();
              for (var i = 0; i < keyList.length; ++i){
                var item = $("<option>");
                if (notYetSelected && ((doShow && keyList[i].toString().toUpperCase() == upperWordStart) || keyList[i].toString() == wordStart)){
                  item.prop('selected', true);
                  notYetSelected = false;
                  selectedValue = data[keyList[i]];
                }
                item.text(data[keyList[i]]);
                item.val(keyList[i]);
                cb.$source.append(item);
              }
            }
            if (doShow){
              cb.refresh();
              cb.clearTarget();
              cb.$target.val(cb.$element.val());
              cb.lookup();
            }
            else{
              if (!notYetSelected){
                cb.$element.val(selectedValue);
              }
            }
          }
        });
      }
      function daInitialize(doScroll){
        daResetCheckinCode();
        daComboBoxes = Object();
        if (daSpinnerTimeout != null){
          clearTimeout(daSpinnerTimeout);
          daSpinnerTimeout = null;
        }
        if (daShowingSpinner){
          daHideSpinner();
        }
        daNotYetScrolled = true;
        // $(".dahelptrigger").click(function(e) {
        //   e.preventDefault();
        //   $(this).tab('show');
        // });
        $(".datableup,.databledown").click(function(e){
          e.preventDefault();
          $(this).blur();
          var row = $(this).parents("tr").first();
          if ($(this).is(".datableup")) {
            var prev = row.prev();
            if (prev.length == 0){
              return false;
            }
            row.addClass("datablehighlighted");
            setTimeout(function(){
              row.insertBefore(prev);
            }, 200);
          }
          else {
            var next = row.next();
            if (next.length == 0){
              return false;
            }
            row.addClass("datablehighlighted");
            setTimeout(function(){
              row.insertAfter(row.next());
            }, 200);
          }
          setTimeout(function(){
            row.removeClass("datablehighlighted");
          }, 1000);
          return false;
        });
        $(".dacollectextra").find('input, textarea, select').prop("disabled", true);
        $(".dacollectextra").find('input.combobox').each(function(){
          daComboBoxes[$(this).attr('id')].disable();
        });
        $("#da-extra-collect").on('click', function(){
          $("<input>").attr({
            type: 'hidden',
            name: '_collect',
            value: $(this).val()
          }).appendTo($("#daform"));
          $("#daform").submit();
          event.preventDefault();
          return false;
        });
        $(".dacollectadd").on('click', function(e){
          e.preventDefault();
          if ($("#daform").valid()){
            var num = $(this).parent().parent().data('collectnum');
            $('div[data-collectnum="' + num + '"]').show('fast');
            $('div[data-collectnum="' + num + '"]').find('input, textarea, select').prop("disabled", false);
            $('div[data-collectnum="' + num + '"]').find('input.combobox').each(function(){
               daComboBoxes[$(this).attr('id')].enable();
            });
            $(this).parent().find("button.dacollectremove").removeClass("dainvisible");
            $(this).parent().find("span.dacollectnum").removeClass("dainvisible");
            $(this).addClass("dainvisible");
            $(".da-first-delete").removeClass("dainvisible");
            rationalizeListCollect();
            var elem = $('div[data-collectnum="' + num + '"]').find('input, textarea, select').first();
            if ($(elem).visible()){
              $(elem).focus();
            }
          }
          return false;
        });
        $("#dasigform").on('submit', daSignatureSubmit);
        $(".dacollectremove").on('click', function(e){
          e.preventDefault();
          var num = $(this).parent().parent().data('collectnum');
          $('div[data-collectnum="' + num + '"]:not(.dacollectextraheader, .dacollectheader, .dacollectfirstheader)').hide('fast');
          $('div[data-collectnum="' + num + '"]').find('input, textarea, select').prop("disabled", true);
          $('div[data-collectnum="' + num + '"]').find('input.combobox').each(function(){
            daComboBoxes[$(this).attr('id')].disable();
          });
          $(this).parent().find("button.dacollectadd").removeClass("dainvisible");
          $(this).parent().find("span.dacollectnum").addClass("dainvisible");
          $(this).addClass("dainvisible");
          rationalizeListCollect();
          return false;
        });
        $(".dacollectremoveexisting").on('click', function(e){
          e.preventDefault();
          var num = $(this).parent().parent().data('collectnum');
          $('div[data-collectnum="' + num + '"]:not(.dacollectextraheader, .dacollectheader, .dacollectfirstheader)').hide('fast');
          $('div[data-collectnum="' + num + '"]').find('input, textarea, select').prop("disabled", true);
          $('div[data-collectnum="' + num + '"]').find('input.combobox').each(function(){
            daComboBoxes[$(this).attr('id')].disable();
          });
          $(this).parent().find("button.dacollectunremove").removeClass("dainvisible");
          $(this).parent().find("span.dacollectremoved").removeClass("dainvisible");
          $(this).addClass("dainvisible");
          rationalizeListCollect();
          return false;
        });
        $(".dacollectunremove").on('click', function(e){
          e.preventDefault();
          var num = $(this).parent().parent().data('collectnum');
          $('div[data-collectnum="' + num + '"]').show('fast');
          $('div[data-collectnum="' + num + '"]').find('input, textarea, select').prop("disabled", false);
          $('div[data-collectnum="' + num + '"]').find('input.combobox').each(function(){
            daComboBoxes[$(this).attr('id')].enable();
          });
          $(this).parent().find("button.dacollectremoveexisting").removeClass("dainvisible");
          $(this).parent().find("button.dacollectremove").removeClass("dainvisible");
          $(this).parent().find("span.dacollectnum").removeClass("dainvisible");
          $(this).parent().find("span.dacollectremoved").addClass("dainvisible");
          $(this).addClass("dainvisible");
          rationalizeListCollect();
          return false;
        });
        //$('#daquestionlabel').click(function(e) {
        //  e.preventDefault();
        //  $(this).tab('show');
        //});
        //$('#dapagetitle').click(function(e) {
        //  if ($(this).prop('href') == '#'){
        //    e.preventDefault();
        //    //$('#daquestionlabel').tab('show');
        //  }
        //});
        $('select.damultiselect').each(function(){
          var varname = atob($(this).data('varname'));
          var theSelect = this;
          $(this).find('option').each(function(){
            var theVal = atob($(this).data('valname'));
            var key = varname + '["' + theVal + '"]';
            if (!daVarLookupSelect[key]){
              daVarLookupSelect[key] = [];
            }
            daVarLookupSelect[key].push({'select': theSelect, 'option': this});
            key = varname + "['" + theVal + "']"
            if (!daVarLookupSelect[key]){
              daVarLookupSelect[key] = [];
            }
            daVarLookupSelect[key].push({'select': theSelect, 'option': this});
          });
        })
        $('.dacurrency').each(function(){
          var theVal = $(this).val().toString();
          if (theVal.indexOf('.') >= 0 || theVal.indexOf(',') >= 0){
            var num = parseFloat(theVal);
            var cleanNum = num.toFixed(""" + str(daconfig.get('currency decimal places', 2)) + """);
            $(this).val(cleanNum);
          }
        });
        $('.dacurrency').on('blur', function(){
          var theVal = $(this).val().toString();
          if (theVal.indexOf('.') >= 0 || theVal.indexOf(',') >= 0){
            var num = parseFloat(theVal);
            var cleanNum = num.toFixed(""" + str(daconfig.get('currency decimal places', 2)) + """);
            if (cleanNum != 'NaN') {
              $(this).val(cleanNum);
            }
          }
        });
        if (navigator.userAgent.match(/(iPad|iPhone|iPod touch);/i)) {
          var selects = document.querySelectorAll("select");
          for (var i = 0; i < selects.length; i++){
            selects[i].appendChild(document.createElement("optgroup"));
          }
        }
        $(".da-to-labelauty").labelauty({ class: "labelauty da-active-invisible dafullwidth" });
        $(".da-to-labelauty-icon").labelauty({ label: false });
        $("button").on('click', function(){
          daWhichButton = this;
          return true;
        });
        $('#dasource').on('shown.bs.collapse', function (e) {
          if (daJsEmbed){
            var scrollTarget = $("#dasource").first().position().top - 60;
            $(daTargetDiv).animate({
              scrollTop: scrollTarget
            }, 1000);
          }
          else{
            var scrollTarget = $("#dasource").first().offset().top - 60;
            $("html, body").animate({
              scrollTop: scrollTarget
            }, 1000);
          }
        });
        $('button[data-bs-target="#dahelp"]').on('shown.bs.tab', function (e) {
          daShowingHelp = 1;
          if (daNotYetScrolled){
            daScrollChatFast();
            daNotYetScrolled = false;
          }""" + debug_readability_help + """
        });
        $('button[data-bs-target="#daquestion"]').on('shown.bs.tab', function (e) {
          daShowingHelp = 0;""" + debug_readability_question + """
        });
        $("input.danota-checkbox").click(function(){
          $(this).parent().find('input.danon-nota-checkbox').each(function(){
            var existing_val = $(this).prop('checked');
            $(this).prop('checked', false);
            if (existing_val != false){
              $(this).trigger('change');
            }
          });
        });
        $("input.danon-nota-checkbox").click(function(){
          $(this).parent().find('input.danota-checkbox').each(function(){
            var existing_val = $(this).prop('checked');
            $(this).prop('checked', false);
            if (existing_val != false){
              $(this).trigger('change');
            }
          });
        });
        $("input.dafile").fileinput({theme: "fas", language: document.documentElement.lang});
        $('select.combobox').combobox();
        $('select.da-ajax-combobox').combobox({clearIfNoMatch: true});
        $('input.da-ajax-combobox').each(function(){
          var cb = daComboBoxes[$(this).attr("id")];
          daFetchAjax(this, cb, false);
          $(this).on('keyup', function(e){
            switch(e.keyCode){
              case 40:
              case 39: // right arrow
              case 38: // up arrow
              case 37: // left arrow
              case 36: // home
              case 35: // end
              case 16: // shift
              case 17: // ctrl
              case 9:  // tab
              case 13: // enter
              case 27: // escape
              case 18: // alt
                return;
            }
            daFetchAjax(this, cb, true);
            daFetchAcceptIncoming = true;
            e.preventDefault();
            return false;
          });
        });
        $("#daemailform").validate({'submitHandler': daValidationHandler, 'rules': {'_attachment_email_address': {'minlength': 1, 'required': true, 'email': true}}, 'messages': {'_attachment_email_address': {'required': """ + json.dumps(word("An e-mail address is required.")) + """, 'email': """ + json.dumps(word("You need to enter a complete e-mail address.")) + """}}, 'errorClass': 'da-has-error invalid-feedback'});
        $("a[data-embaction]").click(daEmbeddedAction);
        $("a[data-js]").click(daEmbeddedJs);
        $("a.da-review-action").click(daReviewAction);
        $("input.dainput-embedded").on('keyup', daAdjustInputWidth);
        $("input.dainput-embedded").each(daAdjustInputWidth);
        var daPopoverTriggerList = [].slice.call(document.querySelectorAll('[data-bs-toggle="popover"]'));
        var daPopoverList = daPopoverTriggerList.map(function (daPopoverTriggerEl) {
          return new bootstrap.Popover(daPopoverTriggerEl, {trigger: """ + json.dumps(interview_status.question.interview.options.get('popover trigger', 'focus')) + """, html: true});
        });
        $('label a[data-bs-toggle="popover"]').on('click', function(event){
          event.preventDefault();
          event.stopPropagation();
          var thePopover = bootstrap.Popover.getOrCreateInstance(this);
          thePopover.show();
          return false;
        });
        if (daPhoneAvailable){
          $("#daPhoneAvailable").removeClass("dainvisible");
        }
        $(".daquestionbackbutton").on('click', function(event){
          event.preventDefault();
          $("#dabackbutton").submit();
          return false;
        });
        $("#dabackbutton").on('submit', function(event){
          if (daShowingHelp){
            event.preventDefault();
            $('#daquestionlabel').tab('show');
            return false;
          }
          $("#dabackbutton").addClass("dabackiconpressed");
          var informed = '';
          if (daInformedChanged){
            informed = '&informed=' + Object.keys(daInformed).join(',');
          }
          var url;
          if (daJsEmbed){
            url = daPostURL;
          }
          else{
            url = $("#dabackbutton").attr('action');
          }
          $.ajax({
            type: "POST",
            url: url,
            beforeSend: addCsrfHeader,
            xhrFields: {
              withCredentials: true
            },
            data: $("#dabackbutton").serialize() + '&ajax=1' + informed,
            success: function(data){
              setTimeout(function(){
                daProcessAjax(data, document.getElementById('backbutton'), 1);
              }, 0);
            },
            error: function(xhr, status, error){
              setTimeout(function(){
                daProcessAjaxError(xhr, status, error);
              }, 0);
            }
          });
          daSpinnerTimeout = setTimeout(daShowSpinner, 1000);
          event.preventDefault();
        });
        $("#daChatOnButton").click(daRingChat);
        $("#daChatOffButton").click(daCloseChat);
        $('#daMessage').bind('keypress keydown keyup', function(e){
          var theCode = e.which || e.keyCode;
          if(theCode == 13) { daSender(); e.preventDefault(); }
        });
        $('#daform button[type="submit"]').click(function(){
          daSubmitter = this;
          document.activeElement.blur();
          return true;
        });
        $('#daform input[type="submit"]').click(function(){
          daSubmitter = this;
          document.activeElement.blur();
          return true;
        });
        $('#daemailform button[type="submit"]').click(function(){
          daSubmitter = this;
          return true;
        });
        $('#dadownloadform button[type="submit"]').click(function(){
          daSubmitter = this;
          return true;
        });
        $(".danavlinks a.daclickable").click(function(e){
          var the_key = $(this).data('key');
          da_action_perform("_da_priority_action", {_action: the_key});
          e.preventDefault();
          return false;
        });
        $(".danav-vertical .danavnested").each(function(){
          var box = this;
          var prev = $(this).prev();
          if (prev && !prev.hasClass('active')){
            var toggler;
            if ($(box).hasClass('danotshowing')){
              toggler = $('<a href="#" class="toggler" role="button" aria-pressed="false">');
              $('<i class="fas fa-caret-right">').appendTo(toggler);
            }
            else{
              toggler = $('<a href="#" class="toggler" role="button" aria-pressed="true">');
              $('<i class="fas fa-caret-down">').appendTo(toggler);
            }
            toggler.appendTo(prev);
            toggler.on('click', function(e){
              var oThis = this;
              $(this).find("svg").each(function(){
                if ($(this).attr('data-icon') == 'caret-down'){
                  $(this).removeClass('fa-caret-down');
                  $(this).addClass('fa-caret-right');
                  $(this).attr('data-icon', 'caret-right');
                  $(box).hide();
                  $(oThis).attr('aria-pressed', 'false');
                  $(box).toggleClass('danotshowing');
                }
                else if ($(this).attr('data-icon') == 'caret-right'){
                  $(this).removeClass('fa-caret-right');
                  $(this).addClass('fa-caret-down');
                  $(this).attr('data-icon', 'caret-down');
                  $(box).show();
                  $(oThis).attr('aria-pressed', 'true');
                  $(box).toggleClass('danotshowing');
                }
              });
              e.stopPropagation();
              e.preventDefault();
              return false;
            });
          }
        });
        $("body").focus();
        if (!daJsEmbed){
          setTimeout(function(){
            var firstInput = $("#daform .da-field-container").not(".da-field-container-note").first().find("input, textarea, select").filter(":visible").first();
            if (firstInput.length > 0 && $(firstInput).visible()){
              $(firstInput).focus();
              var inputType = $(firstInput).attr('type');
              if ($(firstInput).prop('tagName') != 'SELECT' && inputType != "checkbox" && inputType != "radio" && inputType != "hidden" && inputType != "submit" && inputType != "file" && inputType != "range" && inputType != "number" && inputType != "date" && inputType != "time"){
                var strLength = $(firstInput).val().length * 2;
                if (strLength > 0){
                  try {
                    $(firstInput)[0].setSelectionRange(strLength, strLength);
                  }
                  catch(err) {
                    console.log(err.message);
                  }
                }
              }
            }
            else {
              var firstButton = $("#danavbar-collapse .nav-link").filter(':visible').first();
              if (firstButton.length > 0 && $(firstButton).visible()){
                setTimeout(function(){
                  $(firstButton).focus();
                  $(firstButton).blur();
                }, 0);
              }
            }
          }, 15);
        }
        $(".dauncheckspecificothers").on('change', function(){
          if ($(this).is(":checked")){
            var theIds = $.parseJSON(atob($(this).data('unchecklist')));
            var n = theIds.length;
            for (var i = 0; i < n; ++i){
              var elem = document.getElementById(theIds[i]);
              $(elem).prop("checked", false);
              $(elem).trigger('change');
            }
          }
        });
        $(".dauncheckspecificothers").each(function(){
          var theIds = $.parseJSON(atob($(this).data('unchecklist')));
          var n = theIds.length;
          var oThis = this;
          for (var i = 0; i < n; ++i){
            var elem = document.getElementById(theIds[i]);
            $(elem).on('change', function(){
              if ($(this).is(":checked")){
                $(oThis).prop("checked", false);
                $(oThis).trigger('change');
              }
            });
          }
        });
        $(".dauncheckothers").on('change', function(){
          if ($(this).is(":checked")){
            $(".dauncheckable").prop("checked", false);
            $(".dauncheckable").trigger('change');
          }
        });
        $(".dauncheckable").on('change', function(){
          if ($(this).is(":checked")){
            $(".dauncheckothers").prop("checked", false);
            $(".dauncheckothers").trigger('change');
          }
        });
        var navMain = $("#danavbar-collapse");
        navMain.on("click", "a", null, function () {
          if (!($(this).hasClass("dropdown-toggle"))){
            navMain.collapse('hide');
          }
        });
        $("button[data-bs-target='#dahelp']").on("shown.bs.tab", function(){
          if (daJsEmbed){
            $(daTargetDiv)[0].scrollTo(0, 1);
          }
          else{
            window.scrollTo(0, 1);
          }
          $("#dahelptoggle").removeClass('daactivetext');
          $("#dahelptoggle").blur();
        });
        $("#dasourcetoggle").on("click", function(){
          $(this).parent().toggleClass("active");
          $(this).blur();
        });
        $('#dabackToQuestion').click(function(event){
          $('#daquestionlabel').tab('show');
        });
        daVarLookup = Object();
        daVarLookupRev = Object();
        daVarLookupMulti = Object();
        daVarLookupRevMulti = Object();
        if ($("input[name='_varnames']").length){
          the_hash = $.parseJSON(atob($("input[name='_varnames']").val()));
          for (var key in the_hash){
            if (the_hash.hasOwnProperty(key)){
              daVarLookup[the_hash[key]] = key;
              daVarLookupRev[key] = the_hash[key];
              if (!daVarLookupMulti.hasOwnProperty(the_hash[key])){
                daVarLookupMulti[the_hash[key]] = [];
              }
              daVarLookupMulti[the_hash[key]].push(key);
              if (!daVarLookupRevMulti.hasOwnProperty(key)){
                daVarLookupRevMulti[key] = [];
              }
              daVarLookupRevMulti[key].push(the_hash[key]);
            }
          }
        }
        if ($("input[name='_checkboxes']").length){
          var patt = new RegExp(/\[B['"][^\]]*['"]\]$/);
          var pattRaw = new RegExp(/\[R['"][^\]]*['"]\]$/);
          the_hash = $.parseJSON(atob($("input[name='_checkboxes']").val()));
          for (var key in the_hash){
            if (the_hash.hasOwnProperty(key)){
              var checkboxName = atob(key);
              var baseName = checkboxName;
              if (patt.test(baseName)){
                bracketPart = checkboxName.replace(/^.*(\[B?['"][^\]]*['"]\])$/, "$1");
                checkboxName = checkboxName.replace(/^.*\[B?['"]([^\]]*)['"]\]$/, "$1");
                baseName = baseName.replace(/^(.*)\[.*/, "$1");
                var transBaseName = baseName;
                if (($("[name='" + key + "']").length == 0) && (typeof daVarLookup[btoa(transBaseName).replace(/[\\n=]/g, '')] != "undefined")){
                  transBaseName = atob(daVarLookup[btoa(transBaseName).replace(/[\\n=]/g, '')]);
                }
                var convertedName;
                try {
                  convertedName = atob(checkboxName);
                }
                catch (e) {
                  continue;
                }
                var daNameOne = btoa(transBaseName + bracketPart).replace(/[\\n=]/g, '');
                var daNameTwo = btoa(baseName + "['" + convertedName + "']").replace(/[\\n=]/g, '');
                var daNameThree = btoa(baseName + '["' + convertedName + '"]').replace(/[\\n=]/g, '');
                daVarLookupRev[daNameOne] = daNameTwo;
                daVarLookup[daNameTwo] = daNameOne;
                daVarLookup[daNameThree] = daNameOne;
                if (!daVarLookupRevMulti.hasOwnProperty(daNameOne)){
                  daVarLookupRevMulti[daNameOne] = [];
                }
                daVarLookupRevMulti[daNameOne].push(daNameTwo);
                if (!daVarLookupMulti.hasOwnProperty(daNameTwo)){
                  daVarLookupMulti[daNameTwo] = [];
                }
                daVarLookupMulti[daNameTwo].push(daNameOne);
                if (!daVarLookupMulti.hasOwnProperty(daNameThree)){
                  daVarLookupMulti[daNameThree] = [];
                }
                daVarLookupMulti[daNameThree].push(daNameOne);
              }
              else if (pattRaw.test(baseName)){
                bracketPart = checkboxName.replace(/^.*(\[R?['"][^\]]*['"]\])$/, "$1");
                checkboxName = checkboxName.replace(/^.*\[R?['"]([^\]]*)['"]\]$/, "$1");
                baseName = baseName.replace(/^(.*)\[.*/, "$1");
                var transBaseName = baseName;
                if (($("[name='" + key + "']").length == 0) && (typeof daVarLookup[btoa(transBaseName).replace(/[\\n=]/g, '')] != "undefined")){
                  transBaseName = atob(daVarLookup[btoa(transBaseName).replace(/[\\n=]/g, '')]);
                }
                var convertedName;
                try {
                  convertedName = atob(checkboxName);
                }
                catch (e) {
                  continue;
                }
                var daNameOne = btoa(transBaseName + bracketPart).replace(/[\\n=]/g, '');
                var daNameTwo = btoa(baseName + "[" + convertedName + "]").replace(/[\\n=]/g, '')
                daVarLookupRev[daNameOne] = daNameTwo;
                daVarLookup[daNameTwo] = daNameOne;
                if (!daVarLookupRevMulti.hasOwnProperty(daNameOne)){
                  daVarLookupRevMulti[daNameOne] = [];
                }
                daVarLookupRevMulti[daNameOne].push(daNameTwo);
                if (!daVarLookupMulti.hasOwnProperty(daNameTwo)){
                  daVarLookupMulti[daNameTwo] = [];
                }
                daVarLookupMulti[daNameTwo].push(daNameOne);
              }
            }
          }
        }
        daShowIfInProcess = true;
        var daTriggerQueries = [];
        function daOnlyUnique(value, index, self){
          return self.indexOf(value) === index;
        }
        $(".dajsshowif").each(function(){
          var showIfDiv = this;
          var jsInfo = JSON.parse(atob($(this).data('jsshowif')));
          var showIfSign = jsInfo['sign'];
          var showIfMode = jsInfo['mode'];
          var jsExpression = jsInfo['expression'];
          var n = jsInfo['vars'].length;
          for (var i = 0; i < n; ++i){
            var showIfVars = [];
            var initShowIfVar = btoa(jsInfo['vars'][i]).replace(/[\\n=]/g, '');
            var initShowIfVarEscaped = initShowIfVar.replace(/(:|\.|\[|\]|,|=)/g, "\\\\$1");
            var elem = $("[name='" + initShowIfVarEscaped + "']");
            if (elem.length > 0){
              showIfVars.push(initShowIfVar);
            }
            if (daVarLookupMulti.hasOwnProperty(initShowIfVar)){
              for (var j = 0; j < daVarLookupMulti[initShowIfVar].length; j++){
                var altShowIfVar = daVarLookupMulti[initShowIfVar][j];
                var altShowIfVarEscaped = altShowIfVar.replace(/(:|\.|\[|\]|,|=)/g, "\\\\$1");
                var altElem = $("[name='" + altShowIfVarEscaped + "']");
                if (altElem.length > 0 && !$.contains(this, altElem[0])){
                  showIfVars.push(altShowIfVar);
                }
              }
            }
            if (showIfVars.length == 0){
              console.log("ERROR: reference to non-existent field " + jsInfo['vars'][i]);
            }
            for (var j = 0; j < showIfVars.length; ++j){
              var showIfVar = showIfVars[j];
              var showIfVarEscaped = showIfVar.replace(/(:|\.|\[|\]|,|=)/g, "\\\\$1");
              var showHideDiv = function(speed){
                var elem = daGetField(jsInfo['vars'][i]);
                if (elem != null && !$(elem).parents('.da-form-group').first().is($(this).parents('.da-form-group').first())){
                  return;
                }
                var resultt = eval(jsExpression);
                if(resultt){
                  if (showIfSign){
                    if ($(showIfDiv).data('isVisible') != '1'){
                      daShowHideHappened = true;
                    }
                    if (showIfMode == 0){
                      $(showIfDiv).show(speed);
                    }
                    $(showIfDiv).data('isVisible', '1');
                    $(showIfDiv).find('input, textarea, select').prop("disabled", false);
                    $(showIfDiv).find('input.combobox').each(function(){
                      daComboBoxes[$(this).attr('id')].enable();
                    });
                  }
                  else{
                    if ($(showIfDiv).data('isVisible') != '0'){
                      daShowHideHappened = true;
                    }
                    if (showIfMode == 0){
                      $(showIfDiv).hide(speed);
                    }
                    $(showIfDiv).data('isVisible', '0');
                    $(showIfDiv).find('input, textarea, select').prop("disabled", true);
                    $(showIfDiv).find('input.combobox').each(function(){
                      daComboBoxes[$(this).attr('id')].disable();
                    });
                  }
                }
                else{
                  if (showIfSign){
                    if ($(showIfDiv).data('isVisible') != '0'){
                      daShowHideHappened = true;
                    }
                    if (showIfMode == 0){
                      $(showIfDiv).hide(speed);
                    }
                    $(showIfDiv).data('isVisible', '0');
                    $(showIfDiv).find('input, textarea, select').prop("disabled", true);
                    $(showIfDiv).find('input.combobox').each(function(){
                      daComboBoxes[$(this).attr('id')].disable();
                    });
                  }
                  else{
                    if ($(showIfDiv).data('isVisible') != '1'){
                      daShowHideHappened = true;
                    }
                    if (showIfMode == 0){
                      $(showIfDiv).show(speed);
                    }
                    $(showIfDiv).data('isVisible', '1');
                    $(showIfDiv).find('input, textarea, select').prop("disabled", false);
                    $(showIfDiv).find('input.combobox').each(function(){
                      daComboBoxes[$(this).attr('id')].enable();
                    });
                  }
                }
                var daThis = this;
                if (!daShowIfInProcess){
                  daShowIfInProcess = true;
                  $(":input").not("[type='file']").each(function(){
                    if (this != daThis){
                      $(this).trigger('change');
                    }
                  });
                  daShowIfInProcess = false;
                }
              };
              var showHideDivImmediate = function(){
                showHideDiv.apply(this, [null]);
              }
              var showHideDivFast = function(){
                showHideDiv.apply(this, ['fast']);
              }
              daTriggerQueries.push("#" + showIfVarEscaped);
              daTriggerQueries.push("input[type='radio'][name='" + showIfVarEscaped + "']");
              daTriggerQueries.push("input[type='checkbox'][name='" + showIfVarEscaped + "']");
              $("#" + showIfVarEscaped).change(showHideDivFast);
              $("input[type='radio'][name='" + showIfVarEscaped + "']").change(showHideDivFast);
              $("input[type='checkbox'][name='" + showIfVarEscaped + "']").change(showHideDivFast);
              $("#" + showIfVarEscaped).on('daManualTrigger', showHideDivImmediate);
              $("input[type='radio'][name='" + showIfVarEscaped + "']").on('daManualTrigger', showHideDivImmediate);
              $("input[type='checkbox'][name='" + showIfVarEscaped + "']").on('daManualTrigger', showHideDivImmediate);
            }
          }
        });
        $(".dashowif").each(function(){
          var showIfVars = [];
          var showIfSign = $(this).data('showif-sign');
          var showIfMode = parseInt($(this).data('showif-mode'));
          var initShowIfVar = $(this).data('showif-var');
          var varName = atob(initShowIfVar);
          var initShowIfVarEscaped = initShowIfVar.replace(/(:|\.|\[|\]|,|=)/g, "\\\\$1");
          var elem = $("[name='" + initShowIfVarEscaped + "']");
          if (elem.length > 0){
            showIfVars.push(initShowIfVar);
          }
          if (daVarLookupMulti.hasOwnProperty(initShowIfVar)){
            var n = daVarLookupMulti[initShowIfVar].length;
            for (var i = 0; i < n; i++){
              var altShowIfVar = daVarLookupMulti[initShowIfVar][i];
              var altShowIfVarEscaped = altShowIfVar.replace(/(:|\.|\[|\]|,|=)/g, "\\\\$1");
              var altElem = $("[name='" + altShowIfVarEscaped + "']");
              if (altElem.length > 0 && !$.contains(this, altElem[0])){
                showIfVars.push(altShowIfVar);
              }
            }
          }
          var showIfVal = $(this).data('showif-val');
          var saveAs = $(this).data('saveas');
          var showIfDiv = this;
          var n = showIfVars.length;
          for (var i = 0; i < n; ++i){
            var showIfVar = showIfVars[i];
            var showIfVarEscaped = showIfVar.replace(/(:|\.|\[|\]|,|=)/g, "\\\\$1");
            var showHideDiv = function(speed){
              var elem = daGetField(varName, showIfDiv);
              if (elem != null && !$(elem).parents('.da-form-group').first().is($(this).parents('.da-form-group').first())){
                return;
              }
              var theVal;
              var showifParents = $(this).parents(".dashowif");
              if (showifParents.length !== 0 && !($(showifParents[0]).data("isVisible") == '1')){
                theVal = '';
                //console.log("Setting theVal to blank.");
              }
              else if ($(this).attr('type') == "checkbox"){
                theVal = $("input[name='" + showIfVarEscaped + "']:checked").val();
                if (typeof(theVal) == 'undefined'){
                  //console.log('manually setting checkbox value to False');
                  theVal = 'False';
                }
              }
              else if ($(this).attr('type') == "radio"){
                theVal = $("input[name='" + showIfVarEscaped + "']:checked").val();
                if (typeof(theVal) == 'undefined'){
                  theVal = '';
                }
                else if (theVal != '' && $("input[name='" + showIfVarEscaped + "']:checked").hasClass("daobject")){
                  try{
                    theVal = atob(theVal);
                  }
                  catch(e){
                  }
                }
              }
              else{
                theVal = $(this).val();
                if (theVal != '' && $(this).hasClass("daobject")){
                  try{
                    theVal = atob(theVal);
                  }
                  catch(e){
                  }
                }
              }
              //console.log("this is " + $(this).attr('id') + " and saveAs is " + atob(saveAs) + " and showIfVar is " + atob(showIfVar) + " and val is " + String(theVal) + " and showIfVal is " + String(showIfVal));
              if(daShowIfCompare(theVal, showIfVal)){
                if (showIfSign){
                  if ($(showIfDiv).data('isVisible') != '1'){
                    daShowHideHappened = true;
                  }
                  if (showIfMode == 0){
                    $(showIfDiv).show(speed);
                  }
                  $(showIfDiv).data('isVisible', '1');
                  var firstChild = $(showIfDiv).children()[0];
                  if (!$(firstChild).hasClass('dacollectextra') || $(firstChild).is(":visible")){
                    $(showIfDiv).find('input, textarea, select').prop("disabled", false);
                    $(showIfDiv).find('input.combobox').each(function(){
                      daComboBoxes[$(this).attr('id')].enable();
                    });
                  }
                }
                else{
                  if ($(showIfDiv).data('isVisible') != '0'){
                    daShowHideHappened = true;
                  }
                  if (showIfMode == 0){
                    $(showIfDiv).hide(speed);
                  }
                  $(showIfDiv).data('isVisible', '0');
                  $(showIfDiv).find('input, textarea, select').prop("disabled", true);
                  $(showIfDiv).find('input.combobox').each(function(){
                    daComboBoxes[$(this).attr('id')].disable();
                  });
                }
              }
              else{
                if (showIfSign){
                  if ($(showIfDiv).data('isVisible') != '0'){
                    daShowHideHappened = true;
                  }
                  if (showIfMode == 0){
                    $(showIfDiv).hide(speed);
                  }
                  $(showIfDiv).data('isVisible', '0');
                  $(showIfDiv).find('input, textarea, select').prop("disabled", true);
                  $(showIfDiv).find('input.combobox').each(function(){
                    daComboBoxes[$(this).attr('id')].disable();
                  });
                }
                else{
                  if ($(showIfDiv).data('isVisible') != '1'){
                    daShowHideHappened = true;
                  }
                  if (showIfMode == 0){
                    $(showIfDiv).show(speed);
                  }
                  $(showIfDiv).data('isVisible', '1');
                  var firstChild = $(showIfDiv).children()[0];
                  if (!$(firstChild).hasClass('dacollectextra') || $(firstChild).is(":visible")){
                    $(showIfDiv).find('input, textarea, select').prop("disabled", false);
                    $(showIfDiv).find('input.combobox').each(function(){
                      daComboBoxes[$(this).attr('id')].enable();
                    });
                  }
                }
              }
              var daThis = this;
              if (!daShowIfInProcess){
                daShowIfInProcess = true;
                $(":input").not("[type='file']").each(function(){
                  if (this != daThis){
                    $(this).trigger('change');
                  }
                });
                daShowIfInProcess = false;
              }
            };
            var showHideDivImmediate = function(){
              showHideDiv.apply(this, [null]);
            }
            var showHideDivFast = function(){
              showHideDiv.apply(this, ['fast']);
            }
            daTriggerQueries.push("#" + showIfVarEscaped);
            daTriggerQueries.push("input[type='radio'][name='" + showIfVarEscaped + "']");
            daTriggerQueries.push("input[type='checkbox'][name='" + showIfVarEscaped + "']");
            $("#" + showIfVarEscaped).change(showHideDivFast);
            $("#" + showIfVarEscaped).on('daManualTrigger', showHideDivImmediate);
            $("input[type='radio'][name='" + showIfVarEscaped + "']").change(showHideDivFast);
            $("input[type='radio'][name='" + showIfVarEscaped + "']").on('daManualTrigger', showHideDivImmediate);
            $("input[type='checkbox'][name='" + showIfVarEscaped + "']").change(showHideDivFast);
            $("input[type='checkbox'][name='" + showIfVarEscaped + "']").on('daManualTrigger', showHideDivImmediate);
          }
        });
        function daTriggerAllShowHides(){
          var daUniqueTriggerQueries = daTriggerQueries.filter(daOnlyUnique);
          var daFirstTime = true;
          var daTries = 0;
          while ((daFirstTime || daShowHideHappened) && ++daTries < 100){
            daShowHideHappened = false;
            daFirstTime = false;
            var n = daUniqueTriggerQueries.length;
            for (var i = 0; i < n; ++i){
              $(daUniqueTriggerQueries[i]).trigger('daManualTrigger');
            }
          }
          if (daTries >= 100){
            console.log("Too many contradictory 'show if' conditions");
          }
        }
        if (daTriggerQueries.length > 0){
          daTriggerAllShowHides();
        }
        $(".danavlink").last().addClass('thelast');
        $(".danavlink").each(function(){
          if ($(this).hasClass('btn') && !$(this).hasClass('danotavailableyet')){
            var the_a = $(this);
            var the_delay = 1000 + 250 * parseInt($(this).data('index'));
            setTimeout(function(){
              $(the_a).removeClass('""" + app.config['BUTTON_STYLE'] + """secondary');
              if ($(the_a).hasClass('active')){
                $(the_a).addClass('""" + app.config['BUTTON_STYLE'] + """success');
              }
              else{
                $(the_a).addClass('""" + app.config['BUTTON_STYLE'] + """warning');
              }
            }, the_delay);
          }
        });
        daShowIfInProcess = false;
        $("#daSend").click(daSender);
        if (daChatAvailable == 'unavailable'){
          daChatStatus = 'off';
        }
        if (daChatAvailable == 'observeonly'){
          daChatStatus = 'observeonly';
        }
        if ((daChatStatus == 'off' || daChatStatus == 'observeonly') && daChatAvailable == 'available'){
          daChatStatus = 'waiting';
        }
        daDisplayChat();
        if (daBeingControlled){
          daShowControl('fast');
        }
        if (daChatStatus == 'ready' || daBeingControlled){
          daInitializeSocket();
        }
        if (daInitialized == false && daCheckinSeconds > 0){ // why was this set to always retrieve the chat log?
          setTimeout(function(){
            //console.log("daInitialize call to chat_log in checkin");
            $.ajax({
              type: 'POST',
              url: """ + "'" + url_for('checkin', i=yaml_filename) + "'" + """,
              beforeSend: addCsrfHeader,
              xhrFields: {
                withCredentials: true
              },
              data: $.param({action: 'chat_log', ajax: '1', csrf_token: daCsrf}),
              success: daChatLogCallback,
              dataType: 'json'
            });
          }, 200);
        }
        if (daInitialized == true){
          //console.log("Publishing from memory");
          $("#daCorrespondence").html('');
          for(var i = 0; i < daChatHistory.length; i++){
            daPublishMessage(daChatHistory[i]);
          }
        }
        if (daChatStatus != 'off'){
          daSendChanges = true;
        }
        else{
          if (daDoAction == null){
            daSendChanges = false;
          }
          else{
            daSendChanges = true;
          }
        }
        if (daSendChanges){
          $("#daform").each(function(){
            $(this).find(':input').change(daPushChanges);
          });
        }
        daInitialized = true;
        daShowingHelp = 0;
        daSubmitter = null;
        setTimeout(function(){
          $("#daflash .alert-success").hide(300, function(){
            $(self).remove();
          });
        }, 3000);
        if (doScroll){
          setTimeout(function () {
            if (daJsEmbed){
              $(daTargetDiv)[0].scrollTo(0, 1);
              if (daSteps > 1){
                $(daTargetDiv)[0].scrollIntoView();
              }
            }
            else{
              window.scrollTo(0, 1);
            }
          }, 20);
        }
        if (daShowingSpinner){
          daHideSpinner();
        }
        if (daCheckinInterval != null){
          clearInterval(daCheckinInterval);
        }
        if (daCheckinSeconds > 0){
          setTimeout(daCheckin, 100);
          daCheckinInterval = setInterval(daCheckin, daCheckinSeconds);
        }
        daShowNotifications();
        if (daUsingGA){
          daPageview();
        }
        if (daUsingSegment){
          daSegmentEvent();
        }
        $(document).trigger('daPageLoad');
      }
      $(document).ready(function(){
        daInitialize(1);
        //console.log("ready: replaceState " + daSteps);
        if (!daJsEmbed && !daIframeEmbed){
          history.replaceState({steps: daSteps}, "", daLocationBar + """ + json.dumps(page_sep) + """ + daSteps);
        }
        var daReloadAfter = """ + str(int(reload_after)) + """;
        if (daReloadAfter > 0){
          daReloader = setTimeout(function(){daRefreshSubmit();}, daReloadAfter);
        }
        window.onpopstate = function(event) {
          if (event.state != null && event.state.steps < daSteps && daAllowGoingBack){
            $("#dabackbutton").submit();
          }
        };
        $( window ).bind('unload', function() {
          daStopCheckingIn();
          if (daSocket != null && daSocket.connected){
            //console.log('Terminating interview socket because window unloaded');
            daSocket.emit('terminate');
          }
        });
        if (daJsEmbed){
          $.ajax({
            type: "POST",
            url: daPostURL,
            beforeSend: addCsrfHeader,
            xhrFields: {
              withCredentials: true
            },
            data: 'csrf_token=' + daCsrf + '&ajax=1',
            success: function(data){
              setTimeout(function(){
                daProcessAjax(data, $("#daform"), 0);
              }, 0);
            },
            error: function(xhr, status, error){
              setTimeout(function(){
                daProcessAjaxError(xhr, status, error);
              }, 0);
            }
          });
        }
      });
      $(window).ready(daUpdateHeight);
      $(window).resize(daUpdateHeight);
      function daUpdateHeight(){
        $(".dagoogleMap").each(function(){
          var size = $( this ).width();
          $( this ).css('height', size);
        });
      }
      $.validator.setDefaults({
        highlight: function(element) {
            $(element).closest('.da-form-group').addClass('da-group-has-error');
            $(element).addClass('is-invalid');
        },
        unhighlight: function(element) {
            $(element).closest('.da-form-group').removeClass('da-group-has-error');
            $(element).removeClass('is-invalid');
        },
        errorElement: 'span',
        errorClass: 'da-has-error invalid-feedback',
        errorPlacement: function(error, element) {
            $(error).addClass('invalid-feedback');
            var elementName = $(element).attr("name");
            var lastInGroup = $.map(daValidationRules['groups'], function(thefields, thename){
              var fieldsArr;
              if (thefields.indexOf(elementName) >= 0) {
                fieldsArr = thefields.split(" ");
                return fieldsArr[fieldsArr.length - 1];
              }
              else {
                return null;
              }
            })[0];
            if (element.hasClass('dainput-embedded')){
              error.insertAfter(element);
            }
            else if (element.hasClass('dafile-embedded')){
              error.insertAfter(element);
            }
            else if (element.hasClass('daradio-embedded')){
              element.parent().append(error);
            }
            else if (element.hasClass('dacheckbox-embedded')){
              element.parent().append(error);
            }
            else if (element.hasClass('dauncheckable') && lastInGroup){
              $("input[name='" + lastInGroup + "']").parent().append(error);
            }
            else if (element.parent().hasClass('combobox-container')){
              error.insertAfter(element.parent());
            }
            else if (element.hasClass('dafile')){
              var fileContainer = $(element).parents(".file-input").first();
              if (fileContainer.length > 0){
                $(fileContainer).append(error);
              }
              else{
                error.insertAfter(element.parent());
              }
            }
            else if (element.parent('.input-group').length) {
              error.insertAfter(element.parent());
            }
            else if (element.hasClass('da-active-invisible')){
              var choice_with_help = $(element).parents(".dachoicewithhelp").first();
              if (choice_with_help.length > 0){
                $(choice_with_help).parent().append(error);
              }
              else{
                element.parent().append(error);
              }
            }
            else if (element.hasClass('danon-nota-checkbox')){
              element.parent().append(error);
            }
            else {
              error.insertAfter(element);
            }
        }
      });
      $.validator.addMethod("datetime", function(a, b){
        return true;
      });
      $.validator.addMethod("ajaxrequired", function(value, element, params){
        var realElement = $("#" + $(element).attr('name') + "combobox");
        var realValue = $(realElement).val();
        if (!$(realElement).parent().is(":visible")){
          return true;
        }
        if (realValue == null || realValue.replace(/\s/g, '') == ''){
          return false;
        }
        return true;
      });
      $.validator.addMethod('checkone', function(value, element, params){
        var number_needed = params[0];
        var css_query = params[1];
        if ($(css_query).length >= number_needed){
          return true;
        }
        else{
          return false;
        }
      });
      $.validator.addMethod('checkatleast', function(value, element, params){
        if ($(element).attr('name') != '_ignore' + params[0]){
          return true;
        }
        if ($('.dafield' + params[0] + ':checked').length >= params[1]){
          return true;
        }
        else{
          return false;
        }
      });
      $.validator.addMethod('checkatmost', function(value, element, params){
        if ($(element).attr('name') != '_ignore' + params[0]){
          return true;
        }
        if ($('.dafield' + params[0] + ':checked').length > params[1]){
          return false;
        }
        else{
          return true;
        }
      });
      $.validator.addMethod('checkexactly', function(value, element, params){
        if ($(element).attr('name') != '_ignore' + params[0]){
          return true;
        }
        if ($('.dafield' + params[0] + ':checked').length != params[1]){
          return false;
        }
        else{
          return true;
        }
      });
      $.validator.addMethod('selectexactly', function(value, element, params){
        if ($(element).find('option:selected').length == params[0]){
          return true;
        }
        else {
          return false;
        }
      });
      $.validator.addMethod('mindate', function(value, element, params){
        if (value == null || value == ''){
          return true;
        }
        try {
          var date = new Date(value);
          var comparator = new Date(params);
          if (date >= comparator) {
            return true;
          }
        } catch (e) {}
        return false;
      });
      $.validator.addMethod('maxdate', function(value, element, params){
        if (value == null || value == ''){
          return true;
        }
        try {
          var date = new Date(value);
          var comparator = new Date(params);
          if (date <= comparator) {
            return true;
          }
        } catch (e) {}
        return false;
      });
      $.validator.addMethod('minmaxdate', function(value, element, params){
        if (value == null || value == ''){
          return true;
        }
        try {
          var date = new Date(value);
          var before_comparator = new Date(params[0]);
          var after_comparator = new Date(params[1]);
          if (date >= before_comparator && date <= after_comparator) {
            return true;
          }
        } catch (e) {}
        return false;
      });
      $.validator.addMethod('maxuploadsize', function(value, element, param){
        try {
          var limit = parseInt(param) - 2000;
          if (limit <= 0){
            return true;
          }
          var maxImageSize;
          if ($(element).data('maximagesize')){
             maxImageSize = (parseInt($(element).data('maximagesize')) * parseInt($(element).data('maximagesize'))) * 2;
          }
          else {
             maxImageSize = 0;
          }
          if ($(element).attr("type") === "file"){
            if (element.files && element.files.length) {
              var totalSize = 0;
              for ( i = 0; i < element.files.length; i++ ) {
                if (maxImageSize > 0 && element.files[i].size > (0.20 * maxImageSize) && element.files[i].type.match(/image.*/) && !(element.files[i].type.indexOf('image/svg') == 0)){
                  totalSize += maxImageSize;
                }
                else {
                  totalSize += element.files[i].size;
                }
              }
              if (totalSize > limit){
                return false;
              }
            }
            return true;
          }
        } catch (e) {}
        return false;
      });"""
        for info in docassemble.base.functions.custom_types.values():
            if isinstance(info['javascript'], str):
                the_js += "\n      try {\n" + indent_by(info['javascript'].strip(), 8).rstrip() + "\n      }\n      catch {\n        console.log('Error with JavaScript code of CustomDataType " + info['class'].__name__ + "');\n      }"
        if interview_status.question.interview.options.get('send question data', False):
            the_js += "\n      daQuestionData = " + json.dumps(interview_status.as_data(user_dict))
        scripts += """
    <script type="text/javascript" charset="utf-8">
""" + the_js + """
    </script>"""
    if interview_status.question.language != '*':
        interview_language = interview_status.question.language
    else:
        interview_language = current_language
    validation_rules = {'rules': {}, 'messages': {}, 'errorClass': 'da-has-error invalid-feedback', 'debug': False}
    interview_status.exit_url = title_info.get('exit url', None)
    interview_status.exit_link = title_info.get('exit link', 'exit')
    interview_status.exit_label = title_info.get('exit label', word('Exit'))
    interview_status.title = title_info.get('full', default_title)
    interview_status.display_title = title_info.get('logo', interview_status.title)
    interview_status.tabtitle = title_info.get('tab', interview_status.title)
    interview_status.short_title = title_info.get('short', title_info.get('full', default_short_title))
    interview_status.display_short_title = title_info.get('short logo', title_info.get('logo', interview_status.short_title))
    interview_status.title_url = title_info.get('title url', None)
    interview_status.title_url_opens_in_other_window = title_info.get('title url opens in other window', True)
    interview_status.nav_item = title_info.get('navigation bar html', '')
    the_main_page_parts = main_page_parts.get(interview_language, main_page_parts.get('*'))
    interview_status.pre = title_info.get('pre', the_main_page_parts['main page pre'])
    interview_status.post = title_info.get('post', the_main_page_parts['main page post'])
    interview_status.footer = title_info.get('footer', the_main_page_parts['main page footer'] or get_part('global footer'))
    if interview_status.footer:
        interview_status.footer = re.sub(r'</?p.*?>', '', str(interview_status.footer), flags=re.IGNORECASE).strip()
        if interview_status.footer == 'off':
            interview_status.footer = ''
    interview_status.submit = title_info.get('submit', the_main_page_parts['main page submit'])
    interview_status.back = title_info.get('back button label', the_main_page_parts['main page back button label'] or interview_status.question.back())
    interview_status.cornerback = title_info.get('corner back button label', the_main_page_parts['main page corner back button label'] or interview_status.question.back())
    bootstrap_theme = interview_status.question.interview.get_bootstrap_theme()
    if not is_ajax:
        social = copy.deepcopy(daconfig['social'])
        if 'social' in interview_status.question.interview.consolidated_metadata and isinstance(interview_status.question.interview.consolidated_metadata['social'], dict):
            populate_social(social, interview_status.question.interview.consolidated_metadata['social'])
        standard_header_start = standard_html_start(interview_language=interview_language, debug=debug_mode, bootstrap_theme=bootstrap_theme, page_title=interview_status.title, social=social, yaml_filename=yaml_filename)
    if interview_status.question.question_type == "signature":
        interview_status.extra_scripts.append('<script>$( document ).ready(function() {daInitializeSignature();});</script>')
        if interview_status.question.interview.options.get('hide navbar', False):
            bodyclass="dasignature navbarhidden"
        else:
            bodyclass="dasignature"
    else:
        if interview_status.question.interview.options.get('hide navbar', False):
            bodyclass="dabody"
        else:
            bodyclass="dabody da-pad-for-navbar"
    if 'cssClass' in interview_status.extras:
        bodyclass += ' ' + re.sub(r'[^A-Za-z0-9\_]+', '-', interview_status.extras['cssClass'])
    elif hasattr(interview_status.question, 'id'):
        bodyclass += ' question-' + re.sub(r'[^A-Za-z0-9]+', '-', interview_status.question.id.lower())
    if interview_status.footer:
        bodyclass += ' da-pad-for-footer'
    if debug_mode:
        interview_status.screen_reader_text = {}
    if 'speak_text' in interview_status.extras and interview_status.extras['speak_text']:
        interview_status.initialize_screen_reader()
        util_language = docassemble.base.functions.get_language()
        util_dialect = docassemble.base.functions.get_dialect()
        question_language = interview_status.question.language
        if len(interview.translations) > 0:
            the_language = util_language
        elif question_language != '*':
            the_language = question_language
        else:
            the_language = util_language
        if voicerss_config and 'language map' in voicerss_config and isinstance(voicerss_config['language map'], dict) and the_language in voicerss_config['language map']:
            the_language = voicerss_config['language map'][the_language]
        if the_language == util_language and util_dialect is not None:
            the_dialect = util_dialect
        elif voicerss_config and 'dialects' in voicerss_config and isinstance(voicerss_config['dialects'], dict) and the_language in voicerss_config['dialects']:
            the_dialect = voicerss_config['dialects'][the_language]
        elif the_language in valid_voicerss_dialects:
            the_dialect = valid_voicerss_dialects[the_language][0]
        else:
            logmessage("index: unable to determine dialect; reverting to default")
            the_language = DEFAULT_LANGUAGE
            the_dialect = DEFAULT_DIALECT
        for question_type in ('question', 'help'):
            for audio_format in ('mp3', 'ogg'):
                interview_status.screen_reader_links[question_type].append([url_for('speak_file', i=yaml_filename, question=interview_status.question.number, digest='XXXTHEXXX' + question_type + 'XXXHASHXXX', type=question_type, format=audio_format, language=the_language, dialect=the_dialect), audio_mimetype_table[audio_format]])
    if (not validated) and the_question.name == interview_status.question.name:
        for def_key, def_val in new_values.items():
            safe_def_key = safeid(def_key)
            if isinstance(def_val, list):
                def_val = '[' + ','.join(def_val) + ']'
            if safe_def_key in all_field_numbers:
                for number in all_field_numbers[safe_def_key]:
                    try:
                        interview_status.defaults[number] = eval(def_val, pre_user_dict)
                    except:
                        pass
            else:
                try:
                    interview_status.other_defaults[def_key] = eval(def_val, pre_user_dict)
                except:
                    pass
        the_field_errors = field_error
    else:
        the_field_errors = None
    # restore this, maybe
    # if next_action_to_set:
    #     interview_status.next_action.append(next_action_to_set)
    if next_action_to_set:
        if 'event_stack' not in user_dict['_internal']:
            user_dict['_internal']['event_stack'] = {}
        session_uid = interview_status.current_info['user']['session_uid']
        if session_uid not in user_dict['_internal']['event_stack']:
            user_dict['_internal']['event_stack'][session_uid] = []
        already_there = False
        for event_item in user_dict['_internal']['event_stack'][session_uid]:
            if event_item['action'] == next_action_to_set['action']:
                already_there = True
                break
        if not already_there:
            user_dict['_internal']['event_stack'][session_uid].insert(0, next_action_to_set)
    if interview_status.question.interview.use_progress_bar and (interview_status.question.progress is None or interview_status.question.progress >= 0):
        the_progress_bar = progress_bar(user_dict['_internal']['progress'], interview_status.question.interview)
    else:
        the_progress_bar = None
    if interview_status.question.interview.use_navigation and user_dict['nav'].visible():
        if interview_status.question.interview.use_navigation_on_small_screens == 'dropdown':
            current_dict = {}
            dropdown_nav_bar = navigation_bar(user_dict['nav'], interview_status.question.interview, wrapper=False, a_class='dropdown-item', hide_inactive_subs=False, always_open=True, return_dict=current_dict)
            if dropdown_nav_bar != '':
                dropdown_nav_bar = '        <div class="col d-md-none text-end">\n          <div class="dropdown">\n            <button class="btn btn-primary dropdown-toggle" type="button" id="daDropdownSections" data-bs-toggle="dropdown" aria-haspopup="true" aria-expanded="false">' + current_dict.get('title', word("Sections")) + '</button>\n            <div class="dropdown-menu" aria-labelledby="daDropdownSections">' + dropdown_nav_bar + '\n          </div>\n          </div>\n        </div>\n'
        else:
            dropdown_nav_bar = ''
        if interview_status.question.interview.use_navigation == 'horizontal':
            if interview_status.question.interview.use_navigation_on_small_screens is not True:
                nav_class = ' d-none d-md-block'
            else:
                nav_class = ''
            the_nav_bar = navigation_bar(user_dict['nav'], interview_status.question.interview, wrapper=False, inner_div_class='nav flex-row justify-content-center align-items-center nav-pills danav danavlinks danav-horiz danavnested-horiz')
            if the_nav_bar != '':
                the_nav_bar = dropdown_nav_bar + '        <div class="col' + nav_class + '">\n          <div class="nav flex-row justify-content-center align-items-center nav-pills danav danavlinks danav-horiz">\n            ' + the_nav_bar + '\n          </div>\n        </div>\n      </div>\n      <div class="row tab-content">\n'
        else:
            if interview_status.question.interview.use_navigation_on_small_screens == 'dropdown':
                if dropdown_nav_bar:
                    horiz_nav_bar = dropdown_nav_bar + '\n      </div>\n      <div class="row tab-content">\n'
                else:
                    horiz_nav_bar = ''
            elif interview_status.question.interview.use_navigation_on_small_screens:
                horiz_nav_bar = navigation_bar(user_dict['nav'], interview_status.question.interview, wrapper=False, inner_div_class='nav flex-row justify-content-center align-items-center nav-pills danav danavlinks danav-horiz danavnested-horiz')
                if horiz_nav_bar != '':
                    horiz_nav_bar = dropdown_nav_bar + '        <div class="col d-md-none">\n          <div class="nav flex-row justify-content-center align-items-center nav-pills danav danavlinks danav-horiz">\n            ' + horiz_nav_bar + '\n          </div>\n        </div>\n      </div>\n      <div class="row tab-content">\n'
            else:
                horiz_nav_bar = ''
            the_nav_bar = navigation_bar(user_dict['nav'], interview_status.question.interview)
        if the_nav_bar != '':
            if interview_status.question.interview.use_navigation == 'horizontal':
                interview_status.using_navigation = 'horizontal'
            else:
                interview_status.using_navigation = 'vertical'
        else:
            interview_status.using_navigation = False
    else:
        the_nav_bar = ''
        interview_status.using_navigation = False
    content = as_html(interview_status, debug_mode, url_for('index', **index_params), validation_rules, the_field_errors, the_progress_bar, steps - user_dict['_internal']['steps_offset'])
    if debug_mode:
        readability = {}
        for question_type in ('question', 'help'):
            if question_type not in interview_status.screen_reader_text:
                continue
            phrase = to_text(interview_status.screen_reader_text[question_type])
            if (not phrase) or len(phrase) < 10:
                phrase = "The sky is blue."
            phrase = re.sub(r'[^A-Za-z 0-9\.\,\?\#\!\%\&\(\)]', r' ', phrase)
            readability[question_type] = [('Flesch Reading Ease', textstat.flesch_reading_ease(phrase)),
                                          ('Flesch-Kincaid Grade Level', textstat.flesch_kincaid_grade(phrase)),
                                          ('Gunning FOG Scale', textstat.gunning_fog(phrase)),
                                          ('SMOG Index', textstat.smog_index(phrase)),
                                          ('Automated Readability Index', textstat.automated_readability_index(phrase)),
                                          ('Coleman-Liau Index', textstat.coleman_liau_index(phrase)),
                                          ('Linsear Write Formula', textstat.linsear_write_formula(phrase)),
                                          ('Dale-Chall Readability Score', textstat.dale_chall_readability_score(phrase)),
                                          ('Readability Consensus', textstat.text_standard(phrase))]
        readability_report = ''
        for question_type in ('question', 'help'):
            if question_type in readability:
                readability_report += '          <div id="dareadability-' + question_type +'"' + (' style="display: none;"' if question_type == 'help' else '') + '>\n'
                if question_type == 'question':
                    readability_report += '            <h3>' + word("Readability of question") + '</h3>\n'
                else:
                    readability_report += '            <h3>' + word("Readability of help text") + '</h3>\n'
                readability_report += '            <table class="table">' + "\n"
                readability_report += '              <tr><th>' + word("Formula") + '</th><th>' + word("Score") + '</th></tr>' + "\n"
                for read_type, value in readability[question_type]:
                    readability_report += '              <tr><td>' + read_type +'</td><td>' + str(value) + "</td></tr>\n"
                readability_report += '            </table>' + "\n"
                readability_report += '          </div>' + "\n"
    if interview_status.using_screen_reader:
        for question_type in ('question', 'help'):
            if question_type not in interview_status.screen_reader_text:
                continue
            phrase = to_text(interview_status.screen_reader_text[question_type])
            if encrypted:
                the_phrase = encrypt_phrase(phrase, secret)
            else:
                the_phrase = pack_phrase(phrase)
            the_hash = MD5Hash(data=phrase).hexdigest()
            content = re.sub(r'XXXTHEXXX' + question_type + 'XXXHASHXXX', the_hash, content)
            existing_entry = db.session.execute(select(SpeakList).filter_by(filename=yaml_filename, key=user_code, question=interview_status.question.number, digest=the_hash, type=question_type, language=the_language, dialect=the_dialect).with_for_update()).scalar()
            if existing_entry:
                if existing_entry.encrypted:
                    existing_phrase = decrypt_phrase(existing_entry.phrase, secret)
                else:
                    existing_phrase = unpack_phrase(existing_entry.phrase)
                if phrase != existing_phrase:
                    logmessage("index: the phrase changed; updating it")
                    existing_entry.phrase = the_phrase
                    existing_entry.upload = None
                    existing_entry.encrypted = encrypted
            else:
                new_entry = SpeakList(filename=yaml_filename, key=user_code, phrase=the_phrase, question=interview_status.question.number, digest=the_hash, type=question_type, language=the_language, dialect=the_dialect, encrypted=encrypted)
                db.session.add(new_entry)
            db.session.commit()
    append_css_urls = []
    if not is_ajax:
        start_output = standard_header_start
        if 'css' in interview_status.question.interview.external_files:
            for packageref, fileref in interview_status.question.interview.external_files['css']:
                the_url = get_url_from_file_reference(fileref, _package=packageref)
                if is_js:
                    append_css_urls.append(the_url)
                if the_url is not None:
                    start_output += "\n" + '    <link href="' + the_url + '" rel="stylesheet">'
                else:
                    logmessage("index: could not find css file " + str(fileref))
        start_output += global_css + additional_css(interview_status)
        if is_js:
            append_javascript += additional_css(interview_status, js_only=True)
        start_output += '\n    <title>' + interview_status.tabtitle + '</title>\n  </head>\n  <body class="' + bodyclass + '">\n  <div id="dabody">\n'
    if interview_status.question.interview.options.get('hide navbar', False):
        output = make_navbar(interview_status, (steps - user_dict['_internal']['steps_offset']), interview_status.question.interview.consolidated_metadata.get('show login', SHOW_LOGIN), user_dict['_internal']['livehelp'], debug_mode, index_params, extra_class='dainvisible')
    else:
        output = make_navbar(interview_status, (steps - user_dict['_internal']['steps_offset']), interview_status.question.interview.consolidated_metadata.get('show login', SHOW_LOGIN), user_dict['_internal']['livehelp'], debug_mode, index_params)
    output += flash_content + '    <div class="container">' + "\n      " + '<div class="row tab-content">' + "\n"
    if the_nav_bar != '':
        if interview_status.using_navigation == 'vertical':
            output += horiz_nav_bar
        output += the_nav_bar
    output += content
    if 'rightText' in interview_status.extras:
        if interview_status.using_navigation == 'vertical':
            output += '          <section id="daright" role="complementary" class="d-none d-lg-block col-lg-3 col-xl-2 daright">\n'
        else:
            if interview_status.question.interview.flush_left:
                output += '          <section id="daright" role="complementary" class="d-none d-lg-block col-lg-6 col-xl-5 daright">\n'
            else:
                output += '          <section id="daright" role="complementary" class="d-none d-lg-block col-lg-3 col-xl-3 daright">\n'
        output += docassemble.base.util.markdown_to_html(interview_status.extras['rightText'], trim=False, status=interview_status) + "\n"
        output += '          </section>\n'
    output += "      </div>\n"
    if interview_status.question.question_type != "signature" and interview_status.post:
        output += '      <div class="row">' + "\n"
        if interview_status.using_navigation == 'vertical':
            output += '        <div class="offset-xl-3 offset-lg-3 offset-md-3 col-lg-6 col-md-9 col-sm-12 daattributions" id="daattributions">\n'
        else:
            if interview_status.question.interview.flush_left:
                output += '        <div class="offset-xl-1 col-xl-5 col-lg-6 col-md-8 col-sm-12 daattributions" id="daattributions">\n'
            else:
                output += '        <div class="offset-xl-3 offset-lg-3 col-xl-6 col-lg-6 offset-md-2 col-md-8 col-sm-12 daattributions" id="daattributions">\n'
        output += interview_status.post
        output += '        </div>\n'
        output += '      </div>' + "\n"
    if len(interview_status.attributions) > 0:
        output += '      <div class="row">' + "\n"
        if interview_status.using_navigation == 'vertical':
            output += '        <div class="offset-xl-3 offset-lg-3 offset-md-3 col-lg-6 col-md-9 col-sm-12 daattributions" id="daattributions">\n'
        else:
            if interview_status.question.interview.flush_left:
                output += '        <div class="offset-xl-1 col-xl-5 col-lg-6 col-md-8 col-sm-12 daattributions" id="daattributions">\n'
            else:
                output += '        <div class="offset-xl-3 offset-lg-3 col-xl-6 col-lg-6 offset-md-2 col-md-8 col-sm-12 daattributions" id="daattributions">\n'
        output += '          <br/><br/><br/><br/><br/><br/><br/>\n'
        for attribution in sorted(interview_status.attributions):
            output += '          <div><p><cite><small>' + docassemble.base.util.markdown_to_html(attribution, status=interview_status, strip_newlines=True, trim=True) + '</small></cite></p></div>\n'
        output += '        </div>\n'
        output += '      </div>' + "\n"
    if debug_mode:
        output += '      <div id="dasource" class="collapse mt-3">' + "\n"
        output += '      <h2 class="visually-hidden">Information for developers</h2>\n'
        output += '      <div class="row">' + "\n"
        output += '        <div class="col-md-12">' + "\n"
        if interview_status.using_screen_reader:
            output += '          <h3>' + word('Plain text of sections') + '</h3>' + "\n"
            for question_type in ('question', 'help'):
                if question_type in interview_status.screen_reader_text:
                    output += '<pre style="white-space: pre-wrap;">' + to_text(interview_status.screen_reader_text[question_type]) + '</pre>\n'
        output += '          <hr>\n'
        output += '          <h3>' + word('Source code for question') + '<a class="float-end btn btn-info" target="_blank" href="' + url_for('get_variables', i=yaml_filename) + '">' + word('Show variables and values') + '</a></h3>' + "\n"
        if interview_status.question.from_source.path != interview.source.path and interview_status.question.from_source.path is not None:
            output += '          <p style="font-weight: bold;"><small>(' + word('from') + ' ' + interview_status.question.from_source.path +")</small></p>\n"
        if (not hasattr(interview_status.question, 'source_code')) or interview_status.question.source_code is None:
            output += '          <p>' + word('unavailable') + '</p>'
        else:
            output += highlight(interview_status.question.source_code, YamlLexer(), HtmlFormatter())
        if len(interview_status.seeking) > 1:
            output += '          <h4>' + word('How question came to be asked') + '</h4>' + "\n"
            output += get_history(interview, interview_status)
        output += '        </div>' + "\n"
        output += '      </div>' + "\n"
        output += '      <div class="row mt-4">' + "\n"
        output += '        <div class="col-md-8 col-lg-6">' + "\n"
        output += readability_report
        output += '        </div>' + "\n"
        output += '      </div>' + "\n"
        output += '      </div>' + "\n"
    output += '    </div>'
    if interview_status.footer:
        output += """
    <footer class=""" + '"' + app.config['FOOTER_CLASS'] + '"' + """>
      <div class="container">
        """ + interview_status.footer + """
      </div>
    </footer>
"""
    if not is_ajax:
        end_output = scripts + global_js + "\n" + indent_by("".join(interview_status.extra_scripts).strip(), 4).rstrip() + "\n  </div>\n  </body>\n</html>"
    key = 'da:html:uid:' + str(user_code) + ':i:' + str(yaml_filename) + ':userid:' + str(the_user_id)
    pipe = r.pipeline()
    pipe.set(key, json.dumps(dict(body=output, extra_scripts=interview_status.extra_scripts, global_css=global_css, extra_css=interview_status.extra_css, browser_title=interview_status.tabtitle, lang=interview_language, bodyclass=bodyclass, bootstrap_theme=bootstrap_theme)))
    pipe.expire(key, 60)
    pipe.execute()
    if user_dict['_internal']['livehelp']['availability'] != 'unavailable':
        inputkey = 'da:input:uid:' + str(user_code) + ':i:' + str(yaml_filename) + ':userid:' + str(the_user_id)
        r.publish(inputkey, json.dumps(dict(message='newpage', key=key)))
    if is_json:
        data = dict(browser_title=interview_status.tabtitle, lang=interview_language, csrf_token=generate_csrf(), steps=steps, allow_going_back=allow_going_back, message_log=docassemble.base.functions.get_message_log(), id_dict=question_id_dict)
        data.update(interview_status.as_data(user_dict))
        if reload_after and reload_after > 0:
            data['reload_after'] = reload_after
        if 'action' in data and data['action'] == 'redirect' and 'url' in data:
            sys.stderr.write("Redirecting because of a redirect action.\n")
            response = redirect(data['url'])
        else:
            response = jsonify(**data)
    elif is_ajax:
        if interview_status.question.checkin is not None:
            do_action = interview_status.question.checkin
        else:
            do_action = None
        if interview_status.question.interview.options.get('send question data', False):
            response = jsonify(action='body', body=output, extra_scripts=interview_status.extra_scripts, extra_css=interview_status.extra_css, browser_title=interview_status.tabtitle, lang=interview_language, bodyclass=bodyclass, reload_after=reload_after, livehelp=user_dict['_internal']['livehelp'], csrf_token=generate_csrf(), do_action=do_action, steps=steps, allow_going_back=allow_going_back, message_log=docassemble.base.functions.get_message_log(), id_dict=question_id_dict, question_data=interview_status.as_data(user_dict))
        else:
            response = jsonify(action='body', body=output, extra_scripts=interview_status.extra_scripts, extra_css=interview_status.extra_css, browser_title=interview_status.tabtitle, lang=interview_language, bodyclass=bodyclass, reload_after=reload_after, livehelp=user_dict['_internal']['livehelp'], csrf_token=generate_csrf(), do_action=do_action, steps=steps, allow_going_back=allow_going_back, message_log=docassemble.base.functions.get_message_log(), id_dict=question_id_dict)
        if response_wrapper:
            response_wrapper(response)
        if return_fake_html:
            fake_up(response, interview_language)
    elif is_js:
        output = the_js + "\n" + append_javascript
        if 'global css' in daconfig:
            for fileref in daconfig['global css']:
                append_css_urls.append(get_url_from_file_reference(fileref))
        if 'global javascript' in daconfig:
            for fileref in daconfig['global javascript']:
                append_script_urls.append(get_url_from_file_reference(fileref))
        if len(append_css_urls) > 0:
            output += """
      var daLink;"""
        for path in append_css_urls:
            output += """
      daLink = document.createElement('link');
      daLink.href = """ + json.dumps(path) + """;
      daLink.rel = "stylesheet";
      document.head.appendChild(daLink);
"""
        if len(append_script_urls) > 0:
            output += """
      var daScript;"""
        for path in append_script_urls:
            output += """
      daScript = document.createElement('script');
      daScript.src = """ + json.dumps(path) + """;
      document.head.appendChild(daScript);
"""
        response = make_response(output.encode('utf-8'), '200 OK')
        response.headers['Content-type'] = 'application/javascript; charset=utf-8'
    else:
        output = start_output + output + end_output
        response = make_response(output.encode('utf-8'), '200 OK')
        response.headers['Content-type'] = 'text/html; charset=utf-8'
        response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
    release_lock(user_code, yaml_filename)
    if 'in error' in session:
        del session['in error']
    if response_wrapper:
        response_wrapper(response)
    return response

def process_set_variable(field_name, user_dict, vars_set, old_values):
    vars_set.add(field_name)
    try:
        old_values[field_name] = eval(field_name, user_dict)
    except:
        pass

def add_permissions_for_field(the_field, interview_status, files_to_process):
    if hasattr(the_field, 'permissions'):
        if the_field.number in interview_status.extras['permissions']:
            permissions = interview_status.extras['permissions'][the_field.number]
            if 'private' in permissions or 'persistent' in permissions:
                for (filename, file_number, mimetype, extension) in files_to_process:
                    attribute_args = {}
                    if 'private' in permissions:
                        attribute_args['private'] = permissions['private']
                    if 'persistent' in permissions:
                        attribute_args['persistent'] = permissions['persistent']
                    file_set_attributes(file_number, **attribute_args)
            if 'allow_users' in permissions:
                for (filename, file_number, mimetype, extension) in files_to_process:
                    allow_user_id = []
                    allow_email = []
                    for item in permissions['allow_users']:
                        if isinstance(item, int):
                            allow_user_id.append(item)
                        else:
                            allow_email.append(item)
                    file_user_access(file_number, allow_user_id=allow_user_id, allow_email=allow_email)
            if 'allow_privileges' in permissions:
                for (filename, file_number, mimetype, extension) in files_to_process:
                    file_privilege_access(file_number, allow=permissions['allow_privileges'])

def fake_up(response, interview_language):
    response.set_data('<!DOCTYPE html><html lang="' + interview_language + '"><head><meta charset="utf-8"><title>Response</title></head><body><pre>ABCDABOUNDARYSTARTABC' + codecs.encode(response.get_data(), 'base64').decode() + 'ABCDABOUNDARYENDABC</pre></body></html>')
    response.headers['Content-type'] = 'text/html; charset=utf-8'

def add_action_to_stack(interview_status, user_dict, action, arguments):
    unique_id = interview_status.current_info['user']['session_uid']
    if 'event_stack' not in user_dict['_internal']:
        user_dict['_internal']['event_stack'] = {}
    if unique_id not in user_dict['_internal']['event_stack']:
        user_dict['_internal']['event_stack'][unique_id] = []
    if len(user_dict['_internal']['event_stack'][unique_id]) > 0 and user_dict['_internal']['event_stack'][unique_id][0]['action'] == action and user_dict['_internal']['event_stack'][unique_id][0]['arguments'] == arguments:
        user_dict['_internal']['event_stack'][unique_id].pop(0)
    user_dict['_internal']['event_stack'][unique_id].insert(0, {'action': action, 'arguments': arguments})

def sub_indices(the_var, the_user_dict):
    try:
        if the_var.startswith('x.') and 'x' in the_user_dict and isinstance(the_user_dict['x'], DAObject):
            the_var = re.sub(r'^x\.', the_user_dict['x'].instanceName + '.', the_var)
        if the_var.startswith('x[') and 'x' in the_user_dict and isinstance(the_user_dict['x'], DAObject):
            the_var = re.sub(r'^x\[', the_user_dict['x'].instanceName + '[', the_var)
        if re.search(r'\[[ijklmn]\]', the_var):
            the_var = re.sub(r'\[([ijklmn])\]', lambda m: '[' + repr(the_user_dict[m.group(1)]) + ']', the_var)
    except KeyError as the_err:
        missing_var = str(the_err)
        raise DAError("Reference to variable " + missing_var + " that was not defined")
    return the_var

def fixstr(data):
    return bytearray(data, encoding='utf-8').decode('utf-8', 'ignore').encode("utf-8")

def get_history(interview, interview_status):
    output = ''
    has_question = bool(hasattr(interview_status, 'question'))
    index = 0
    seeking_len = len(interview_status.seeking)
    if seeking_len:
        starttime = interview_status.seeking[0]['time']
        seen_done = False
        for stage in interview_status.seeking:
            if seen_done:
                output = ''
                seen_done = False
            index += 1
            if index < seeking_len and 'reason' in interview_status.seeking[index] and interview_status.seeking[index]['reason'] in ('asking', 'running') and interview_status.seeking[index]['question'] is stage['question'] and 'question' in stage and 'reason' in stage and stage['reason'] == 'considering':
                continue
            the_time = " at %.5fs" % (stage['time'] - starttime)
            if 'question' in stage and 'reason' in stage and (has_question is False or index < (seeking_len - 1) or stage['question'] is not interview_status.question):
                if stage['reason'] == 'initial':
                    output += "          <h5>Ran initial code" + the_time + "</h5>\n"
                elif stage['reason'] == 'mandatory question':
                    output += "          <h5>Tried to ask mandatory question" + the_time + "</h5>\n"
                elif stage['reason'] == 'mandatory code':
                    output += "          <h5>Tried to run mandatory code" + the_time + "</h5>\n"
                elif stage['reason'] == 'asking':
                    output += "          <h5>Tried to ask question" + the_time + "</h5>\n"
                elif stage['reason'] == 'running':
                    output += "          <h5>Tried to run block" + the_time + "</h5>\n"
                elif stage['reason'] == 'considering':
                    output += "          <h5>Considered using block" + the_time + "</h5>\n"
                elif stage['reason'] == 'objects from file':
                    output += "          <h5>Tried to load objects from file" + the_time + "</h5>\n"
                elif stage['reason'] == 'data':
                    output += "          <h5>Tried to load data" + the_time + "</h5>\n"
                elif stage['reason'] == 'objects':
                    output += "          <h5>Tried to load objects" + the_time + "</h5>\n"
                elif stage['reason'] == 'result of multiple choice':
                    output += "          <h5>Followed the result of multiple choice selection" + the_time + "</h5>\n"
                if stage['question'].from_source.path != interview.source.path and stage['question'].from_source.path is not None:
                    output += '          <p style="font-weight: bold;"><small>(' + word('from') + ' ' + stage['question'].from_source.path +")</small></p>\n"
                if (not hasattr(stage['question'], 'source_code')) or stage['question'].source_code is None:
                    output += word('(embedded question, source code not available)')
                else:
                    output += highlight(stage['question'].source_code, YamlLexer(), HtmlFormatter())
            elif 'variable' in stage:
                output += '          <h5>Needed definition of <code class="da-variable-needed">' + str(stage['variable']) + "</code>" + the_time + "</h5>\n"
            elif 'done' in stage:
                output += "          <h5>Completed processing" + the_time + "</h5>\n"
                seen_done = True
    return output

def is_mobile_or_tablet():
    ua_string = request.headers.get('User-Agent', None)
    if ua_string is not None:
        response = ua_parse(ua_string)
        if response.is_mobile or response.is_tablet:
            return True
    return False

def get_referer():
    return request.referrer or None

def add_referer(user_dict, referer=None):
    if referer:
        user_dict['_internal']['referer'] = referer
    elif request.referrer:
        user_dict['_internal']['referer'] = request.referrer
    else:
        user_dict['_internal']['referer'] = None

@app.template_filter('word')
def word_filter(text):
    return docassemble.base.functions.word(str(text))

def get_part(part, default=None):
    if default is None:
        default = str()
    if part not in page_parts:
        return default
    if 'language' in session:
        lang = session['language']
    else:
        lang = DEFAULT_LANGUAGE
    if lang in page_parts[part]:
        return page_parts[part][lang]
    if lang != DEFAULT_LANGUAGE and DEFAULT_LANGUAGE in page_parts[part]:
        return page_parts[part][DEFAULT_LANGUAGE]
    if '*' in page_parts[part]:
        return page_parts[part]['*']
    return default

@app.context_processor
def utility_processor():
    def user_designator(the_user):
        if the_user.email:
            return the_user.email
        else:
            return re.sub(r'.*\$', '', the_user.social_id)
    if 'language' in session:
        docassemble.base.functions.set_language(session['language'])
        lang = session['language']
    elif 'Accept-Language' in request.headers:
        langs = docassemble.base.functions.parse_accept_language(request.headers['Accept-Language'])
        if len(langs) > 0:
            docassemble.base.functions.set_language(langs[0])
            lang = langs[0]
        else:
            docassemble.base.functions.set_language(DEFAULT_LANGUAGE)
            lang = DEFAULT_LANGUAGE
    else:
        docassemble.base.functions.set_language(DEFAULT_LANGUAGE)
        lang = DEFAULT_LANGUAGE
    def in_debug():
        return DEBUG
    return dict(word=docassemble.base.functions.word, in_debug=in_debug, user_designator=user_designator, get_part=get_part, current_language=lang)

@app.route('/speakfile', methods=['GET'])
def speak_file():
    audio_file = None
    filename = request.args.get('i', None)
    if filename is None:
        abort(400)
    session_info = get_session(filename)
    if session_info is None:
        abort(400)
    key = session_info['uid']
    encrypted = session_info['encrypted']
    question = request.args.get('question', None)
    question_type = request.args.get('type', None)
    file_format = request.args.get('format', None)
    the_language = request.args.get('language', None)
    the_dialect = request.args.get('dialect', None)
    the_hash = request.args.get('digest', None)
    secret = request.cookies.get('secret', None)
    if secret is not None:
        secret = str(secret)
    if file_format not in ('mp3', 'ogg') or not (filename and key and question and question_type and file_format and the_language and the_dialect):
        logmessage("speak_file: could not serve speak file because invalid or missing data was provided: filename " + str(filename) + " and key " + str(key) + " and question number " + str(question) + " and question type " + str(question_type) + " and language " + str(the_language) + " and dialect " + str(the_dialect))
        return ('File not found', 404)
    entry = db.session.execute(select(SpeakList).filter_by(filename=filename, key=key, question=question, digest=the_hash, type=question_type, language=the_language, dialect=the_dialect)).scalar()
    if not entry:
        logmessage("speak_file: could not serve speak file because no entry could be found in speaklist for filename " + str(filename) + " and key " + str(key) + " and question number " + str(question) + " and question type " + str(question_type) + " and language " + str(the_language) + " and dialect " + str(the_dialect))
        return ('File not found', 404)
    if not entry.upload:
        existing_entry = db.session.execute(select(SpeakList).where(and_(SpeakList.phrase == entry.phrase, SpeakList.language == entry.language, SpeakList.dialect == entry.dialect, SpeakList.upload != None, SpeakList.encrypted == entry.encrypted))).scalar()
        if existing_entry:
            logmessage("speak_file: found existing entry: " + str(existing_entry.id) + ".  Setting to " + str(existing_entry.upload))
            entry.upload = existing_entry.upload
        else:
            if not VOICERSS_ENABLED:
                logmessage("speak_file: could not serve speak file because voicerss not enabled")
                return ('File not found', 404)
            new_file_number = get_new_file_number(key, 'speak.mp3', yaml_file_name=filename)
            #phrase = codecs.decode(entry.phrase, 'base64')
            if entry.encrypted:
                phrase = decrypt_phrase(entry.phrase, secret)
            else:
                phrase = unpack_phrase(entry.phrase)
            url = voicerss_config.get('url', "https://api.voicerss.org/")
            #logmessage("Retrieving " + url)
            audio_file = SavedFile(new_file_number, extension='mp3', fix=True, should_not_exist=True)
            audio_file.fetch_url_post(url, dict(f=voicerss_config.get('format', '16khz_16bit_stereo'), key=voicerss_config['key'], src=phrase, hl=str(entry.language) + '-' + str(entry.dialect)))
            if audio_file.size_in_bytes() > 100:
                call_array = [daconfig.get('pacpl', 'pacpl'), '-t', 'ogg', audio_file.path + '.mp3']
                logmessage("speak_file: calling " + " ".join(call_array))
                result = subprocess.run(call_array, check=False).returncode
                if result != 0:
                    logmessage("speak_file: failed to convert downloaded mp3 (" + audio_file.path + '.mp3' + ") to ogg")
                    return ('File not found', 404)
                entry.upload = new_file_number
                audio_file.finalize()
                db.session.commit()
            else:
                logmessage("speak_file: download from voicerss (" + url + ") failed")
                return ('File not found', 404)
    if not entry.upload:
        logmessage("speak_file: upload file number was not set")
        return ('File not found', 404)
    if not audio_file:
        audio_file = SavedFile(entry.upload, extension='mp3', fix=True)
    the_path = audio_file.path + '.' + file_format
    if not os.path.isfile(the_path):
        logmessage("speak_file: could not serve speak file because file (" + the_path + ") not found")
        return ('File not found', 404)
    response = send_file(the_path, mimetype=audio_mimetype_table[file_format])
    return response

def interview_menu(absolute_urls=False, start_new=False, tag=None):
    interview_info = []
    for key, yaml_filename in sorted(daconfig['dispatch'].items()):
        try:
            interview = docassemble.base.interview_cache.get_interview(yaml_filename)
            if interview.is_unlisted():
                continue
            if current_user.is_anonymous:
                if not interview.allowed_to_see_listed(is_anonymous=True):
                    continue
            else:
                if not interview.allowed_to_see_listed(has_roles=[role.name for role in current_user.roles]):
                    continue
            if interview.source is None:
                package = None
            else:
                package = interview.source.get_package()
            titles = interview.get_title(dict(_internal={}))
            tags = interview.get_tags(dict(_internal={}))
            metadata = copy.deepcopy(interview.consolidated_metadata)
            if 'tags' in metadata:
                del metadata['tags']
            interview_title = titles.get('full', titles.get('short', word('Untitled')))
            subtitle = titles.get('sub', None)
            status_class = None
            subtitle_class = None
        except:
            interview_title = yaml_filename
            tags = set()
            metadata = {}
            package = None
            subtitle = None
            status_class = 'dainterviewhaserror'
            subtitle_class = 'dainvisible'
            logmessage("interview_dispatch: unable to load interview file " + yaml_filename)
        if tag is not None and tag not in tags:
            continue
        if absolute_urls:
            if start_new:
                url = url_for('run_interview', dispatch=key, _external=True, reset='1')
            else:
                url = url_for('redirect_to_interview', dispatch=key, _external=True)
        else:
            if start_new:
                url = url_for('run_interview', dispatch=key, reset='1')
            else:
                url = url_for('redirect_to_interview', dispatch=key)
        interview_info.append(dict(link=url, title=interview_title, status_class=status_class, subtitle=subtitle, subtitle_class=subtitle_class, filename=yaml_filename, package=package, tags=sorted(tags), metadata=metadata))
    return interview_info

@app.route('/list', methods=['GET'])
def interview_start():
    if current_user.is_anonymous and not daconfig.get('allow anonymous access', True):
        return redirect(url_for('user.login', next=url_for('interview_start', **request.args)))
    setup_translation()
    if len(daconfig['dispatch']) == 0:
        return redirect(url_for('index', i=final_default_yaml_filename))
    is_json = bool(('json' in request.form and as_int(request.form['json'])) or ('json' in request.args and as_int(request.args['json'])))
    tag = request.args.get('tag', None)
    if daconfig.get('dispatch interview', None) is not None:
        if is_json:
            if tag:
                return redirect(url_for('index', i=daconfig.get('dispatch interview'), from_list='1', json='1', tag=tag))
            else:
                return redirect(url_for('index', i=daconfig.get('dispatch interview'), from_list='1', json='1'))
        else:
            if tag:
                return redirect(url_for('index', i=daconfig.get('dispatch interview'), from_list='1', tag=tag))
            else:
                return redirect(url_for('index', i=daconfig.get('dispatch interview'), from_list='1'))
    if 'embedded' in request.args and int(request.args['embedded']):
        the_page = 'pages/start-embedded.html'
        embed = True
    else:
        embed = False
    interview_info = interview_menu(absolute_urls=embed, tag=tag)
    if is_json:
        return jsonify(action='menu', interviews=interview_info)
    argu = dict(version_warning=None, interview_info=interview_info) #extra_css=Markup(global_css), extra_js=Markup(global_js), tab_title=daconfig.get('start page title', word('Interviews')), title=daconfig.get('start page heading', word('Available interviews'))
    if embed:
        the_page = 'pages/start-embedded.html'
    else:
        if 'start page template' in daconfig and daconfig['start page template']:
            the_page = docassemble.base.functions.package_template_filename(daconfig['start page template'])
            if the_page is None:
                raise DAError("Could not find start page template " + daconfig['start page template'])
            with open(the_page, 'r', encoding='utf-8') as fp:
                template_string = fp.read()
                return render_template_string(template_string, **argu)
        else:
            the_page = 'pages/start.html'
    resp = make_response(render_template(the_page, **argu))
    if embed:
        resp.headers['Access-Control-Allow-Origin'] = '*'
    return resp

@app.route('/start/<package>/<directory>/<filename>/', methods=['GET'])
def redirect_to_interview_in_package_directory(package, directory, filename):
    if COOKIELESS_SESSIONS:
        return html_index()
    arguments = {}
    for arg in request.args:
        arguments[arg] = request.args[arg]
    arguments['i'] = 'docassemble.' + package + ':data/questions/' + directory + '/' + filename + '.yml'
    if 'session' not in arguments:
        arguments['new_session'] = '1'
    request.args = arguments
    return index(refer=['start_directory', package, directory, filename])

@app.route('/start/<package>/<filename>/', methods=['GET'])
def redirect_to_interview_in_package(package, filename):
    if COOKIELESS_SESSIONS:
        return html_index()
    arguments = {}
    for arg in request.args:
        arguments[arg] = request.args[arg]
    if re.search(r'playground[0-9]', package):
        arguments['i'] = 'docassemble.' + package + ':' + filename + '.yml'
    else:
        arguments['i'] = 'docassemble.' + package + ':data/questions/' + filename + '.yml'
    if 'session' not in arguments:
        arguments['new_session'] = '1'
    request.args = arguments
    return index(refer=['start', package, filename])

@app.route('/start/<dispatch>/', methods=['GET'])
def redirect_to_interview(dispatch):
    #logmessage("redirect_to_interview: the dispatch is " + str(dispatch))
    if COOKIELESS_SESSIONS:
        return html_index()
    yaml_filename = daconfig['dispatch'].get(dispatch, None)
    if yaml_filename is None:
        return ('File not found', 404)
    arguments = {}
    for arg in request.args:
        arguments[arg] = request.args[arg]
    arguments['i'] = yaml_filename
    if 'session' not in arguments:
        arguments['new_session'] = '1'
    request.args = arguments
    return index(refer=['start_dispatch', dispatch])

@app.route('/run/<package>/<directory>/<filename>/', methods=['GET'])
def run_interview_in_package_directory(package, directory, filename):
    if COOKIELESS_SESSIONS:
        return html_index()
    arguments = {}
    for arg in request.args:
        arguments[arg] = request.args[arg]
    arguments['i'] = 'docassemble.' + package + ':data/questions/' + directory + '/' + filename + '.yml'
    request.args = arguments
    return index(refer=['run_direcory', package, directory, filename])

@app.route('/run/<package>/<filename>/', methods=['GET'])
def run_interview_in_package(package, filename):
    if COOKIELESS_SESSIONS:
        return html_index()
    arguments = {}
    for arg in request.args:
        arguments[arg] = request.args[arg]
    if re.search(r'playground[0-9]', package):
        arguments['i'] = 'docassemble.' + package + ':' + filename + '.yml'
    else:
        arguments['i'] = 'docassemble.' + package + ':data/questions/' + filename + '.yml'
    request.args = arguments
    return index(refer=['run', package, filename])

@app.route('/run/<dispatch>/', methods=['GET'])
def run_interview(dispatch):
    if COOKIELESS_SESSIONS:
        return html_index()
    yaml_filename = daconfig['dispatch'].get(dispatch, None)
    if yaml_filename is None:
        return ('File not found', 404)
    arguments = {}
    for arg in request.args:
        arguments[arg] = request.args[arg]
    arguments['i'] = yaml_filename
    request.args = arguments
    return index(refer=['run_dispatch', dispatch])

@app.route('/storedfile/<uid>/<number>/<filename>.<extension>', methods=['GET'])
def serve_stored_file(uid, number, filename, extension):
    return do_serve_stored_file(uid, number, filename, extension)

@app.route('/storedfiledownload/<uid>/<number>/<filename>.<extension>', methods=['GET'])
def serve_stored_file_download(uid, number, filename, extension):
    return do_serve_stored_file(uid, number, filename, extension, download=True)

def do_serve_stored_file(uid, number, filename, extension, download=False):
    number = re.sub(r'[^0-9]', '', str(number))
    if not can_access_file_number(number, uids=[uid]):
        return ('File not found', 404)
    try:
        file_info = get_info_from_file_number(number, privileged=True, uids=get_session_uids())
    except:
        return ('File not found', 404)
    if 'path' not in file_info:
        return ('File not found', 404)
    else:
        if not os.path.isfile(file_info['path']):
            return ('File not found', 404)
        response = send_file(file_info['path'], mimetype=file_info['mimetype'])
        if download:
            response.headers['Content-Disposition'] = 'attachment; filename=' + json.dumps(filename + '.' + extension)
        response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
        return response

@app.route('/tempfile/<code>/<filename>.<extension>', methods=['GET'])
def serve_temporary_file(code, filename, extension):
    return do_serve_temporary_file(code, filename, extension)

@app.route('/tempfiledownload/<code>/<filename>.<extension>', methods=['GET'])
def serve_temporary_file_download(code, filename, extension):
    return do_serve_temporary_file(code, filename, extension, download=True)

def do_serve_temporary_file(code, filename, extension, download=False):
    file_info = r.get('da:tempfile:' + str(code))
    if file_info is None:
        logmessage("serve_temporary_file: file_info was none")
        return ('File not found', 404)
    (section, file_number) = file_info.decode().split('^')
    the_file = SavedFile(file_number, fix=True, section=section)
    the_path = the_file.path
    if not os.path.isfile(the_path):
        return ('File not found', 404)
    (extension, mimetype) = get_ext_and_mimetype(filename + '.' + extension)
    response = send_file(the_path, mimetype=mimetype)
    if download:
        response.headers['Content-Disposition'] = 'attachment; filename=' + json.dumps(filename + '.' + extension)
    return response

@app.route('/packagezip', methods=['GET'])
@login_required
@roles_required(['admin', 'developer'])
def download_zip_package():
    package_name = request.args.get('package', None)
    if not package_name:
        return ('File not found', 404)
    package_name = werkzeug.utils.secure_filename(package_name)
    package = db.session.execute(select(Package).filter_by(active=True, name=package_name, type='zip')).scalar()
    if package is None:
        return ('File not found', 404)
    if not current_user.has_role('admin'):
        auth = db.session.execute(select(PackageAuth).filter_by(package_id=package.id, user_id=current_user.id)).scalar()
        if auth is None:
            return ('File not found', 404)
    try:
        file_info = get_info_from_file_number(package.upload, privileged=True)
    except:
        return ('File not found', 404)
    response = send_file(file_info['path'] + '.zip', mimetype='application/zip')
    filename = re.sub(r'\.', '-', package_name) + '.zip'
    response.headers['Content-Disposition'] = 'attachment; filename=' + json.dumps(filename)
    response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
    return response

@app.route('/uploadedfile/<number>/<filename>.<extension>', methods=['GET'])
def serve_uploaded_file_with_filename_and_extension(number, filename, extension):
    return do_serve_uploaded_file_with_filename_and_extension(number, filename, extension)

@app.route('/uploadedfiledownload/<number>/<filename>.<extension>', methods=['GET'])
def serve_uploaded_file_with_filename_and_extension_download(number, filename, extension):
    return do_serve_uploaded_file_with_filename_and_extension(number, filename, extension, download=True)

def do_serve_uploaded_file_with_filename_and_extension(number, filename, extension, download=False):
    filename = secure_filename_spaces_ok(filename)
    extension = werkzeug.utils.secure_filename(extension)
    privileged = bool(current_user.is_authenticated and current_user.has_role('admin', 'advocate'))
    number = re.sub(r'[^0-9]', '', str(number))
    if cloud is not None and daconfig.get('use cloud urls', False):
        if not (privileged or can_access_file_number(number, uids=get_session_uids())):
            return ('File not found', 404)
        the_file = SavedFile(number)
        if download:
            return redirect(the_file.temp_url_for(_attachment=True))
        else:
            return redirect(the_file.temp_url_for())
    else:
        try:
            file_info = get_info_from_file_number(number, privileged=privileged, uids=get_session_uids())
        except:
            return ('File not found', 404)
        if 'path' not in file_info:
            return ('File not found', 404)
        else:
            #logmessage("Filename is " + file_info['path'] + '.' + extension)
            if os.path.isfile(file_info['path'] + '.' + extension):
                #logmessage("Using " + file_info['path'] + '.' + extension)
                extension, mimetype = get_ext_and_mimetype(file_info['path'] + '.' + extension)
                response = send_file(file_info['path'] + '.' + extension, mimetype=mimetype)
                if download:
                    response.headers['Content-Disposition'] = 'attachment; filename=' + json.dumps(filename + '.' + extension)
                response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
                return response
            if os.path.isfile(os.path.join(os.path.dirname(file_info['path']), filename + '.' + extension)):
                #logmessage("Using " + os.path.join(os.path.dirname(file_info['path']), filename + '.' + extension))
                extension, mimetype = get_ext_and_mimetype(filename + '.' + extension)
                response = send_file(os.path.join(os.path.dirname(file_info['path']), filename + '.' + extension), mimetype=mimetype)
                if download:
                    response.headers['Content-Disposition'] = 'attachment; filename=' + json.dumps(filename + '.' + extension)
                response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
                return response
            return ('File not found', 404)

@app.route('/uploadedfile/<number>.<extension>', methods=['GET'])
def serve_uploaded_file_with_extension(number, extension):
    return do_serve_uploaded_file_with_extension(number, extension)

@app.route('/uploadedfiledownload/<number>.<extension>', methods=['GET'])
def serve_uploaded_file_with_extension_download(number, extension):
    return do_serve_uploaded_file_with_extension(number, extension, download=True)

def do_serve_uploaded_file_with_extension(number, extension, download=False):
    extension = werkzeug.utils.secure_filename(extension)
    privileged = bool(current_user.is_authenticated and current_user.has_role('admin', 'advocate'))
    number = re.sub(r'[^0-9]', '', str(number))
    if cloud is not None and daconfig.get('use cloud urls', False):
        if not can_access_file_number(number, uids=get_session_uids()):
            return ('File not found', 404)
        the_file = SavedFile(number)
        if download:
            return redirect(the_file.temp_url_for(_attachment=True))
        else:
            return redirect(the_file.temp_url_for())
    else:
        try:
            file_info = get_info_from_file_number(number, privileged=privileged, uids=get_session_uids())
        except:
            return ('File not found', 404)
        if 'path' not in file_info:
            return ('File not found', 404)
        else:
            if os.path.isfile(file_info['path'] + '.' + extension):
                extension, mimetype = get_ext_and_mimetype(file_info['path'] + '.' + extension)
                response = send_file(file_info['path'] + '.' + extension, mimetype=mimetype)
                if download:
                    response.headers['Content-Disposition'] = 'attachment; filename=' + json.dumps(str(number) + '.' + extension)
                response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
                return response
            else:
                return ('File not found', 404)
    return ('File not found', 404)

@app.route('/uploadedfile/<number>', methods=['GET'])
def serve_uploaded_file(number):
    return do_serve_uploaded_file(number)

def do_serve_uploaded_file(number, download=False):
    number = re.sub(r'[^0-9]', '', str(number))
    privileged = bool(current_user.is_authenticated and current_user.has_role('admin', 'advocate'))
    try:
        file_info = get_info_from_file_number(number, privileged=privileged, uids=get_session_uids())
    except:
        return ('File not found', 404)
    if 'path' not in file_info:
        return ('File not found', 404)
    else:
        if not os.path.isfile(file_info['path']):
            return ('File not found', 404)
        response = send_file(file_info['path'], mimetype=file_info['mimetype'])
        if download:
            response.headers['Content-Disposition'] = 'attachment; filename=' + json.dumps(os.path.basename(file_info['path']))
        response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
        return response
    return ('File not found', 404)

@app.route('/uploadedpage/<number>/<page>', methods=['GET'])
def serve_uploaded_page(number, page):
    return do_serve_uploaded_page(number, page, size='page')

@app.route('/uploadedpagedownload/<number>/<page>', methods=['GET'])
def serve_uploaded_page_download(number, page):
    return do_serve_uploaded_page(number, page, download=True, size='page')

@app.route('/uploadedpagescreen/<number>/<page>', methods=['GET'])
def serve_uploaded_pagescreen(number, page):
    return do_serve_uploaded_page(number, page, size='screen')

@app.route('/uploadedpagescreendownload/<number>/<page>', methods=['GET'])
def serve_uploaded_pagescreen_download(number, page):
    return do_serve_uploaded_page(number, page, download=True, size='screen')

def do_serve_uploaded_page(number, page, download=False, size='page'):
    number = re.sub(r'[^0-9]', '', str(number))
    page = re.sub(r'[^0-9]', '', str(page))
    privileged = bool(current_user.is_authenticated and current_user.has_role('admin', 'advocate'))
    try:
        file_info = get_info_from_file_number(number, privileged=privileged, uids=get_session_uids())
    except Exception as err:
        logmessage("do_serve_uploaded_page: " + err.__class__.__name__ + str(err))
        return ('File not found', 404)
    if 'path' not in file_info:
        logmessage('serve_uploaded_page: no access to file number ' + str(number))
        return ('File not found', 404)
    try:
        the_file = DAFile(mimetype=file_info['mimetype'], extension=file_info['extension'], number=number, make_thumbnail=page)
        filename = the_file.page_path(page, size)
    except Exception as err:
        logmessage("Could not make thumbnail: " + err.__class__.__name__ + ": " + str(err))
        filename = None
    if filename is None:
        logmessage("do_serve_uploaded_page: sending blank image")
        the_file = docassemble.base.functions.package_data_filename('docassemble.base:data/static/blank_page.png')
        response = send_file(the_file, mimetype='image/png')
        response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
        return response
    if os.path.isfile(filename):
        response = send_file(filename, mimetype='image/png')
        if download:
            response.headers['Content-Disposition'] = 'attachment; filename=' + json.dumps(os.path.basename(filename))
        response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
        return response
    else:
        logmessage('do_serve_uploaded_page: path ' + filename + ' is not a file')
        return ('File not found', 404)

@app.route('/visit_interview', methods=['GET', 'POST'])
@login_required
@roles_required(['admin', 'advocate'])
def visit_interview():
    setup_translation()
    i = request.args.get('i', None)
    uid = request.args.get('uid', None)
    userid = request.args.get('userid', None)
    key = 'da:session:uid:' + str(uid) + ':i:' + str(i) + ':userid:' + str(userid)
    try:
        obj = fix_pickle_obj(r.get(key))
    except:
        return ('Interview not found', 404)
    if 'secret' not in obj or 'encrypted' not in obj:
        return ('Interview not found', 404)
    session_info = update_session(i, uid=uid, encrypted=obj['encrypted'])
    if 'user_id' not in session:
        session['user_id'] = current_user.id
    if 'tempuser' in session:
        del session['tempuser']
    response = redirect(url_for('index', i=i))
    response.set_cookie('visitor_secret', obj['secret'], httponly=True, secure=app.config['SESSION_COOKIE_SECURE'], samesite=app.config['SESSION_COOKIE_SAMESITE'])
    return response

@app.route('/observer', methods=['GET', 'POST'])
@login_required
@roles_required(['admin', 'advocate'])
def observer():
    setup_translation()
    session['observer'] = 1
    i = request.args.get('i', None)
    uid = request.args.get('uid', None)
    userid = request.args.get('userid', None)
    observation_script = """
    <script>
      var daMapInfo = null;
      var daWhichButton = null;
      var daSendChanges = false;
      var daNoConnectionCount = 0;
      var daConnected = false;
      var daConfirmed = false;
      var daObserverChangesInterval = null;
      var daInitialized = false;
      var daShowingSpinner = false;
      var daSpinnerTimeout = null;
      var daShowingHelp = false;
      var daInformedChanged = false;
      var daDisable = null;
      var daCsrf = """ + json.dumps(generate_csrf()) + """;
      var daShowIfInProcess = false;
      var daFieldsToSkip = ['_checkboxes', '_empties', '_ml_info', '_back_one', '_files', '_files_inline', '_question_name', '_the_image', '_save_as', '_success', '_datatypes', '_event', '_visible', '_tracker', '_track_location', '_varnames', '_next_action', '_next_action_to_set', 'ajax', 'json', 'informed', 'csrf_token', '_action', '_order_changes', '_collect', '_null_question'];
      var daVarLookup = Object();
      var daVarLookupRev = Object();
      var daVarLookupMulti = Object();
      var daVarLookupRevMulti = Object();
      var daVarLookupSelect = Object();
      var daTargetDiv = "#dabody";
      var daLocationBar = """ + json.dumps(url_for('index', i=i)) + """;
      var daPostURL = """ + json.dumps(url_for('index', i=i, _external=True)) + """;
      var daYamlFilename = """ + json.dumps(i) + """;
      var daGlobalEval = eval;
      var daShowHideHappened = false;
      function daShowSpinner(){
        if ($("#daquestion").length > 0){
          $('<div id="daSpinner" class="da-spinner-container da-top-for-navbar"><div class="container"><div class="row"><div class="col text-center"><span class="da-spinner"><i class="fas fa-spinner fa-spin"><\/i><\/span><\/div><\/div><\/div><\/div>').appendTo(daTargetDiv);
        }
        else{
          var newSpan = document.createElement('span');
          var newI = document.createElement('i');
          $(newI).addClass("fas fa-spinner fa-spin");
          $(newI).appendTo(newSpan);
          $(newSpan).attr("id", "daSpinner");
          $(newSpan).addClass("da-sig-spinner da-top-for-navbar");
          $(newSpan).appendTo("#dasigtoppart");
        }
        daShowingSpinner = true;
      }
      function daHideSpinner(){
        $("#daSpinner").remove();
        daShowingSpinner = false;
        daSpinnerTimeout = null;
      }
      function daDisableIfNotHidden(query, value){
        $(query).each(function(){
          var showIfParent = $(this).parents('.dashowif, .dajsshowif');
          if (!(showIfParent.length && ($(showIfParent[0]).data('isVisible') == '0' || !$(showIfParent[0]).is(":visible")))){
            if ($(this).hasClass('combobox')){
              if (value){
                daComboBoxes[$(this).attr('id')].disable();
              }
              else {
                daComboBoxes[$(this).attr('id')].enable();
              }
            }
            else {
              $(this).prop("disabled", value);
            }
          }
        });
      }
      function daShowIfCompare(theVal, showIfVal){
        if (typeof theVal == 'string' && theVal.match(/^-?\d+\.\d+$/)){
          theVal = parseFloat(theVal);
        }
        else if (typeof theVal == 'string' && theVal.match(/^-?\d+$/)){
          theVal = parseInt(theVal);
        }
        if (typeof showIfVal == 'string' && showIfVal.match(/^-?\d+\.\d+$/)){
          showIfVal = parseFloat(showIfVal);
        }
        else if (typeof showIfVal == 'string' && showIfVal.match(/^-?\d+$/)){
          showIfVal = parseInt(showIfVal);
        }
        if (typeof theVal == 'string' || typeof showIfVal == 'string'){
          if (String(showIfVal) == 'None' && String(theVal) == ''){
            return true;
          }
          return (String(theVal) == String(showIfVal));
        }
        return (theVal == showIfVal);
      }
      function rationalizeListCollect(){
        var finalNum = $(".dacollectextraheader").last().data('collectnum');
        var num = $(".dacollectextraheader:visible").last().data('collectnum');
        if (parseInt(num) < parseInt(finalNum)){
          if ($('div.dacollectextraheader[data-collectnum="' + num + '"]').find(".dacollectadd").hasClass('dainvisible')){
            $('div.dacollectextraheader[data-collectnum="' + (num + 1) + '"]').show('fast');
          }
        }
        var n = parseInt(finalNum);
        var firstNum = parseInt($(".dacollectextraheader").first().data('collectnum'));
        while (n-- > firstNum){
          if ($('div.dacollectextraheader[data-collectnum="' + (n + 1) + '"]:visible').length > 0){
            if (!$('div.dacollectextraheader[data-collectnum="' + (n + 1) + '"]').find(".dacollectadd").hasClass('dainvisible') && $('div.dacollectextraheader[data-collectnum="' + n + '"]').find(".dacollectremove").hasClass('dainvisible')){
              $('div.dacollectextraheader[data-collectnum="' + (n + 1) + '"]').hide();
            }
          }
        }
        var n = parseInt(finalNum);
        var seenAddAnother = false;
        while (n-- > firstNum){
          if ($('div.dacollectextraheader[data-collectnum="' + (n + 1) + '"]:visible').length > 0){
            if (!$('div.dacollectextraheader[data-collectnum="' + (n + 1) + '"]').find(".dacollectadd").hasClass('dainvisible')){
              seenAddAnother = true;
            }
            var current = $('div.dacollectextraheader[data-collectnum="' + n + '"]');
            if (seenAddAnother && !$(current).find(".dacollectadd").hasClass('dainvisible')){
              $(current).find(".dacollectadd").addClass('dainvisible');
              $(current).find(".dacollectunremove").removeClass('dainvisible');
            }
          }
        }
      }
      var daNotificationContainer = """ + json.dumps(NOTIFICATION_CONTAINER) + """;
      var daNotificationMessage = """ + json.dumps(NOTIFICATION_MESSAGE) + """;
      Object.defineProperty(String.prototype, "daSprintf", {
        value: function () {
          var args = Array.from(arguments),
            i = 0;
          function defaultNumber(iValue) {
            return iValue != undefined && !isNaN(iValue) ? iValue : "0";
          }
          function defaultString(iValue) {
            return iValue == undefined ? "" : "" + iValue;
          }
          return this.replace(
            /%%|%([+\\-])?([^1-9])?(\\d+)?(\\.\\d+)?([deEfhHioQqs])/g,
            function (match, sign, filler, scale, precision, type) {
              var strOut, space, value;
              var asNumber = false;
              if (match == "%%") return "%";
              if (i >= args.length) return match;
              value = args[i];
              while (Array.isArray(value)) {
                args.splice(i, 1);
                for (var j = i; value.length > 0; j++)
                  args.splice(j, 0, value.shift());
                value = args[i];
              }
              i++;
              if (filler == undefined) filler = " "; // default
              if (scale == undefined && !isNaN(filler)) {
                scale = filler;
                filler = " ";
              }
              if (sign == undefined) sign = "sqQ".indexOf(type) >= 0 ? "+" : "-"; // default
              if (scale == undefined) scale = 0; // default
              if (precision == undefined) precision = ".0"; // default
              scale = parseInt(scale);
              precision = parseInt(precision.substr(1));
              switch (type) {
                case "d":
                case "i":
                  // decimal integer
                  asNumber = true;
                  strOut = parseInt(defaultNumber(value));
                  if (precision > 0) strOut += "." + "0".repeat(precision);
                  break;
                case "e":
                case "E":
                  // float in exponential notation
                  asNumber = true;
                  strOut = parseFloat(defaultNumber(value));
                  if (precision == 0) strOut = strOut.toExponential();
                  else strOut = strOut.toExponential(precision);
                  if (type == "E") strOut = strOut.replace("e", "E");
                  break;
                case "f":
                  // decimal float
                  asNumber = true;
                  strOut = parseFloat(defaultNumber(value));
                  if (precision != 0) strOut = strOut.toFixed(precision);
                  break;
                case "o":
                case "h":
                case "H":
                  // Octal or Hexagesimal integer notation
                  strOut =
                    "\\\\" +
                    (type == "o" ? "0" : type) +
                    parseInt(defaultNumber(value)).toString(type == "o" ? 8 : 16);
                  break;
                case "q":
                  // single quoted string
                  strOut = "'" + defaultString(value) + "'";
                  break;
                case "Q":
                  // double quoted string
                  strOut = '"' + defaultString(value) + '"';
                  break;
                default:
                  // string
                  strOut = defaultString(value);
                  break;
              }
              if (typeof strOut != "string") strOut = "" + strOut;
              if ((space = strOut.length) < scale) {
                if (asNumber) {
                  if (sign == "-") {
                    if (strOut.indexOf("-") < 0)
                      strOut = filler.repeat(scale - space) + strOut;
                    else
                      strOut =
                        "-" +
                        filler.repeat(scale - space) +
                        strOut.replace("-", "");
                  } else {
                    if (strOut.indexOf("-") < 0)
                      strOut = "+" + filler.repeat(scale - space - 1) + strOut;
                    else
                      strOut =
                        "-" +
                        filler.repeat(scale - space) +
                        strOut.replace("-", "");
                  }
                } else {
                  if (sign == "-") strOut = filler.repeat(scale - space) + strOut;
                  else strOut = strOut + filler.repeat(scale - space);
                }
              } else if (asNumber && sign == "+" && strOut.indexOf("-") < 0)
                strOut = "+" + strOut;
              return strOut;
            }
          );
        },
      });
      Object.defineProperty(window, "daSprintf", {
        value: function (str, ...rest) {
          if (typeof str == "string")
            return String.prototype.daSprintf.apply(str, rest);
          return "";
        },
      });
      function daGoToAnchor(target){
        scrollTarget = $(target).first().offset().top - 60;
        if (scrollTarget != null){
          $("html, body").animate({
            scrollTop: scrollTarget
          }, 500);
        }
      }
      function dabtoa(str) {
        return window.btoa(str).replace(/[\\n=]/g, '');
      }
      function daatob(str) {
        return window.atob(str);
      }
      function getFields(){
        var allFields = [];
        for (var rawFieldName in daVarLookup){
          if (daVarLookup.hasOwnProperty(rawFieldName)){
            var fieldName = atob(rawFieldName);
            if (allFields.indexOf(fieldName) == -1){
              allFields.push(fieldName);
            }
          }
        }
        return allFields;
      }
      var daGetFields = getFields;
      function getField(fieldName, notInDiv){
        if (daVarLookupSelect[fieldName]){
          var n = daVarLookupSelect[fieldName].length;
          for (var i = 0; i < n; ++i){
            var elem = daVarLookupSelect[fieldName][i].select;
            if (!$(elem).prop('disabled')){
              var showifParents = $(elem).parents(".dajsshowif,.dashowif");
              if (showifParents.length == 0 || $(showifParents[0]).data("isVisible") == '1'){
                if (notInDiv && $.contains(notInDiv, elem)){
                  continue;
                }
                return elem;
              }
            }
          }
        }
        var fieldNameEscaped = dabtoa(fieldName);
        var possibleElements = [];
        daAppendIfExists(fieldNameEscaped, possibleElements);
        if (daVarLookupMulti.hasOwnProperty(fieldNameEscaped)){
          for (var i = 0; i < daVarLookupMulti[fieldNameEscaped].length; ++i){
            daAppendIfExists(daVarLookupMulti[fieldNameEscaped][i], possibleElements);
          }
        }
        var returnVal = null;
        for (var i = 0; i < possibleElements.length; ++i){
          if (!$(possibleElements[i]).prop('disabled')){
            var showifParents = $(possibleElements[i]).parents(".dajsshowif,.dashowif");
            if (showifParents.length == 0 || $(showifParents[0]).data("isVisible") == '1'){
              if (notInDiv && $.contains(notInDiv, possibleElements[i])){
                continue;
              }
              returnVal = possibleElements[i];
            }
          }
        }
        return returnVal;
      }
      var daGetField = getField;
      function setField(fieldName, val){
        var elem = daGetField(fieldName);
        if (elem == null){
          console.log('setField: reference to non-existent field ' + fieldName);
          return;
        }
        if ($(elem).attr('type') == "checkbox"){
          if (val){
            if ($(elem).prop('checked') != true){
              $(elem).prop('checked', true);
              $(elem).trigger('change');
            }
          }
          else{
            if ($(elem).prop('checked') != false){
              $(elem).prop('checked', false);
              $(elem).trigger('change');
            }
          }
        }
        else if ($(elem).attr('type') == "radio"){
          var fieldNameEscaped = $(elem).attr('name').replace(/(:|\.|\[|\]|,|=)/g, "\\\\$1");
          var wasSet = false;
          $("input[name='" + fieldNameEscaped + "']").each(function(){
            if ($(this).val() == val){
              if ($(this).prop('checked') != true){
                $(this).prop('checked', true);
                $(this).trigger('change');
              }
              wasSet = true;
              return false;
            }
          });
          if (!wasSet){
            console.log('setField: could not set radio button ' + fieldName + ' to ' + val);
          }
        }
        else{
          if ($(elem).val() != val){
            $(elem).val(val);
            $(elem).trigger('change');
          }
        }
      }
      var daSetField = setField;
      function val(fieldName){
        var elem = getField(fieldName);
        if (elem == null){
          return null;
        }
        var showifParents = $(elem).parents(".dajsshowif");
        if (showifParents.length !== 0 && !($(showifParents[0]).data("isVisible") == '1')){
          theVal = null;
        }
        else if ($(elem).attr('type') == "checkbox"){
          if ($(elem).prop('checked')){
            theVal = true;
          }
          else{
            theVal = false;
          }
        }
        else if ($(elem).attr('type') == "radio"){
          var fieldNameEscaped = $(elem).attr('name').replace(/(:|\.|\[|\]|,|=)/g, "\\\\$1");
          theVal = $("input[name='" + fieldNameEscaped + "']:checked").val();
          if (typeof(theVal) == 'undefined'){
            theVal = null;
          }
          else{
            if (theVal == 'True'){
              theVal = true;
            }
            else if (theVal == 'False'){
              theVal = false;
            }
          }
        }
        else{
          theVal = $(elem).val();
        }
        return theVal;
      }
      var da_val = val;
      window.daTurnOnControl = function(){
        //console.log("Turning on control");
        daSendChanges = true;
        daNoConnectionCount = 0;
        daResetPushChanges();
        daSocket.emit('observerStartControl', {uid: """ + json.dumps(uid) + """, i: """ + json.dumps(i) + """, userid: """ + json.dumps(str(userid)) + """});
      }
      window.daTurnOffControl = function(){
        //console.log("Turning off control");
        if (!daSendChanges){
          //console.log("Already turned off");
          return;
        }
        daSendChanges = false;
        daConfirmed = false;
        daStopPushChanges();
        daSocket.emit('observerStopControl', {uid: """ + json.dumps(uid) + """, i: """ + json.dumps(i) + """, userid: """ + json.dumps(str(userid)) + """});
        return;
      }
      function daInjectTrim(handler){
        return function (element, event) {
          if (element.tagName === "TEXTAREA" || (element.tagName === "INPUT" && element.type !== "password" && element.type !== "date" && element.type !== "datetime" && element.type !== "file")) {
            setTimeout(function(){
              element.value = $.trim(element.value);
            }, 10);
          }
          return handler.call(this, element, event);
        };
      }
      function daInvalidHandler(form, validator){
        var errors = validator.numberOfInvalids();
        var scrollTarget = null;
        if (errors && $(validator.errorList[0].element).parents('.da-form-group').length > 0) {
          if (daJsEmbed){
            scrollTarget = $(validator.errorList[0].element).parents('.da-form-group').first().position().top - 60;
          }
          else{
            scrollTarget = $(validator.errorList[0].element).parents('.da-form-group').first().offset().top - 60;
          }
        }
        if (scrollTarget != null){
          if (daJsEmbed){
            $(daTargetDiv).animate({
              scrollTop: scrollTarget
            }, 1000);
          }
          else{
            $("html, body").animate({
              scrollTop: scrollTarget
            }, 1000);
          }
        }
      }
      function daValidationHandler(form){
        //console.log("observer: daValidationHandler");
        return(false);
      }
      function daStopPushChanges(){
        if (daObserverChangesInterval != null){
          clearInterval(daObserverChangesInterval);
        }
      }
      function daResetPushChanges(){
        if (daObserverChangesInterval != null){
          clearInterval(daObserverChangesInterval);
        }
        daObserverChangesInterval = setInterval(daPushChanges, """ + str(CHECKIN_INTERVAL) + """);
      }
      function daPushChanges(){
        //console.log("Pushing changes");
        if (daObserverChangesInterval != null){
          clearInterval(daObserverChangesInterval);
        }
        if (!daSendChanges || !daConnected){
          return;
        }
        daObserverChangesInterval = setInterval(daPushChanges, """ + str(CHECKIN_INTERVAL) + """);
        daSocket.emit('observerChanges', {uid: """ + json.dumps(uid) + """, i: """ + json.dumps(i) + """, userid: """ + json.dumps(str(userid)) + """, parameters: JSON.stringify($("#daform").serializeArray())});
      }
      function daProcessAjaxError(xhr, status, error){
        if (xhr.responseType == undefined || xhr.responseType == '' || xhr.responseType == 'text'){
          var theHtml = xhr.responseText;
          if (theHtml == undefined){
            $(daTargetDiv).html("error");
          }
          else{
            theHtml = theHtml.replace(/<script[^>]*>[^<]*<\/script>/g, '');
            $(daTargetDiv).html(theHtml);
          }
          if (daJsEmbed){
            $(daTargetDiv)[0].scrollTo(0, 1);
          }
          else{
            window.scrollTo(0, 1);
          }
        }
        else {
          console.log("daProcessAjaxError: response was not text");
        }
      }
      function daAddScriptToHead(src){
        var head = document.getElementsByTagName("head")[0];
        var script = document.createElement("script");
        script.type = "text/javascript";
        script.src = src;
        script.async = true;
        script.defer = true;
        head.appendChild(script);
      }
      function daSubmitter(event){
        if (!daSendChanges || !daConnected){
          event.preventDefault();
          return false;
        }
        var theAction = null;
        if ($(this).hasClass('da-review-action')){
          theAction = $(this).data('action');
        }
        var embeddedJs = $(this).data('js');
        var embeddedAction = $(this).data('embaction');
        var linkNum = $(this).data('linknum');
        var theId = $(this).attr('id');
        if (theId == 'dapagetitle'){
          theId = 'daquestionlabel';
        }
        var theName = $(this).attr('name');
        var theValue = $(this).val();
        var skey;
        if (linkNum){
          skey = 'a[data-linknum="' + linkNum + '"]';
        }
        else if (embeddedAction){
          skey = 'a[data-embaction="' + embeddedAction.replace(/(:|\.|\[|\]|,|=|\/|\")/g, '\\\\$1') + '"]';
        }
        else if (theAction){
          skey = 'a[data-action="' + theAction.replace(/(:|\.|\[|\]|,|=|\/|\")/g, '\\\\$1') + '"]';
        }
        else if (theId){
          skey = '#' + theId.replace(/(:|\.|\[|\]|,|=|\/|\")/g, '\\\\$1');
        }
        else if (theName){
          skey = '#' + $(this).parents("form").attr('id') + ' ' + $(this).prop('tagName').toLowerCase() + '[name="' + theName.replace(/(:|\.|\[|\]|,|=|\/)/g, '\\\\$1') + '"]';
          if (typeof theValue !== 'undefined'){
            skey += '[value="' + theValue + '"]'
          }
        }
        else{
          skey = '#' + $(this).parents("form").attr('id') + ' ' + $(this).prop('tagName').toLowerCase() + '[type="submit"]';
        }
        //console.log("Need to click on " + skey);
        if (daObserverChangesInterval != null && embeddedJs == null && theId != "dabackToQuestion" && theId != "dahelptoggle" && theId != "daquestionlabel"){
          clearInterval(daObserverChangesInterval);
        }
        daSocket.emit('observerChanges', {uid: """ + json.dumps(uid) + """, i: """ + json.dumps(i) + """, userid: """ + json.dumps(str(userid)) + """, clicked: skey, parameters: JSON.stringify($("#daform").serializeArray())});
        if (embeddedJs != null){
          //console.log("Running the embedded js");
          daGlobalEval(decodeURIComponent(embeddedJs));
        }
        if (theId != "dabackToQuestion" && theId != "dahelptoggle" && theId != "daquestionlabel"){
          event.preventDefault();
          return false;
        }
      }
      function daAdjustInputWidth(e){
        var contents = $(this).val();
        var leftBracket = new RegExp('<', 'g');
        var rightBracket = new RegExp('>', 'g');
        contents = contents.replace(/&/g,'&amp;').replace(leftBracket,'&lt;').replace(rightBracket,'&gt;').replace(/ /g, '&nbsp;');
        $('<span class="dainput-embedded" id="dawidth">').html( contents ).appendTo('#daquestion');
        $("#dawidth").css('min-width', $(this).css('min-width'));
        $("#dawidth").css('background-color', $(daTargetDiv).css('background-color'));
        $("#dawidth").css('color', $(daTargetDiv).css('background-color'));
        $(this).width($('#dawidth').width() + 16);
        setTimeout(function(){
          $("#dawidth").remove();
        }, 0);
      }
      function daShowHelpTab(){
          //$('#dahelptoggle').tab('show');
      }
      function flash(message, priority, clear){
        if (priority == null){
          priority = 'info'
        }
        if (!$("#daflash").length){
          $(daTargetDiv).append(daSprintf(daNotificationContainer, ""));
        }
        if (clear){
          $("#daflash").empty();
        }
        if (message != null){
          $("#daflash").append(daSprintf(daNotificationMessage, priority, message));
          if (priority == 'success'){
            setTimeout(function(){
              $("#daflash .alert-success").hide(300, function(){
                $(this).remove();
              });
            }, 3000);
          }
        }
      }
      var da_flash = flash;
      function JSON_stringify(s){
         var json = JSON.stringify(s);
         return json.replace(/[\\u007f-\\uffff]/g,
            function(c) {
              return '\\\\u'+('0000'+c.charCodeAt(0).toString(16)).slice(-4);
            }
         );
      }
      function url_action(action, args){
        //redo?
        if (args == null){
            args = {};
        }
        data = {action: action, arguments: args};
        var url;
        if (daJsEmbed){
          url = daPostURL + "&action=" + encodeURIComponent(btoa(JSON_stringify(data)))
        }
        else{
          url = daLocationBar + "&action=" + encodeURIComponent(btoa(JSON_stringify(data)))
        }
        return url;
      }
      var da_url_action = url_action;
      function action_call(action, args, callback){
        //redo?
        if (args == null){
            args = {};
        }
        if (callback == null){
            callback = function(){};
        }
        var data = {action: action, arguments: args};
        var url;
        if (daJsEmbed){
          url = daPostURL + "&action=" + encodeURIComponent(btoa(JSON_stringify(data)))
        }
        else{
          url = daLocationBar + "&action=" + encodeURIComponent(btoa(JSON_stringify(data)))
        }
        return $.ajax({
          type: "GET",
          url: url,
          success: callback,
          error: function(xhr, status, error){
            setTimeout(function(){
              daProcessAjaxError(xhr, status, error);
            }, 0);
          }
        });
      }
      var da_action_call = action_call;
      var url_action_call = action_call;
      function action_perform(action, args){
        //redo
        if (args == null){
            args = {};
        }
        var data = {action: action, arguments: args};
        daSpinnerTimeout = setTimeout(daShowSpinner, 1000);
        return $.ajax({
          type: "POST",
          url: daLocationBar,
          data: $.param({_action: btoa(JSON_stringify(data)), csrf_token: daCsrf, ajax: 1}),
          success: function(data){
            setTimeout(function(){
              daProcessAjax(data, $("#daform"), 1);
            }, 0);
          },
          error: function(xhr, status, error){
            setTimeout(function(){
              daProcessAjaxError(xhr, status, error);
            }, 0);
          },
          dataType: 'json'
        });
      }
      var da_action_perform = action_perform;
      var url_action_perform = action_perform;
      function action_perform_with_next(action, args, next_data){
        //redo
        //console.log("action_perform_with_next: " + action + " | " + next_data)
        if (args == null){
            args = {};
        }
        var data = {action: action, arguments: args};
        daSpinnerTimeout = setTimeout(daShowSpinner, 1000);
        return $.ajax({
          type: "POST",
          url: daLocationBar,
          data: $.param({_action: btoa(JSON_stringify(data)), _next_action_to_set: btoa(JSON_stringify(next_data)), csrf_token: daCsrf, ajax: 1}),
          success: function(data){
            setTimeout(function(){
              daProcessAjax(data, $("#daform"), 1);
            }, 0);
          },
          error: function(xhr, status, error){
            setTimeout(function(){
              daProcessAjaxError(xhr, status, error);
            }, 0);
          },
          dataType: 'json'
        });
      }
      var da_action_perform_with_next = action_perform_with_next;
      var url_action_perform_with_next = action_perform_with_next;
      function get_interview_variables(callback){
        if (callback == null){
          callback = function(){};
        }
        return $.ajax({
          type: "GET",
          url: """ + '"' + url_for('get_variables', i=i) + '"' + """,
          success: callback,
          error: function(xhr, status, error){
            setTimeout(function(){
              daProcessAjaxError(xhr, status, error);
            }, 0);
          }
        });
      }
      var da_get_interview_variables = get_interview_variables;
      function daInitialize(doScroll){
        if (daSpinnerTimeout != null){
          clearTimeout(daSpinnerTimeout);
          daSpinnerTimeout = null;
        }
        if (daShowingSpinner){
          daHideSpinner();
        }
        $('button[type="submit"], input[type="submit"], a.da-review-action, #dabackToQuestion, #daquestionlabel, #dapagetitle, #dahelptoggle, a[data-linknum], a[data-embaction], #dabackbutton').click(daSubmitter);
        $(".da-to-labelauty").labelauty({ class: "labelauty da-active-invisible dafullwidth" });
        //$(".da-to-labelauty-icon").labelauty({ label: false });
        var navMain = $("#danavbar-collapse");
        navMain.on("click", "a", null, function () {
          if (!($(this).hasClass("dropdown-toggle"))){
            navMain.collapse('hide');
          }
        });
        var daPopoverTriggerList = [].slice.call(document.querySelectorAll('[data-bs-toggle="popover"]'));
        var daPopoverList = daPopoverTriggerList.map(function (daPopoverTriggerEl) {
          return new bootstrap.Popover(daPopoverTriggerEl, {trigger: "focus", html: true});
        });
        $("input.danota-checkbox").click(function(){
          $(this).parent().find('input.danon-nota-checkbox').each(function(){
            if ($(this).prop('checked') != false){
              $(this).prop('checked', false);
              $(this).trigger('change');
            }
          });
        });
        $("input.danon-nota-checkbox").click(function(){
          $(this).parent().find('input.danota-checkbox').each(function(){
            if ($(this).prop('checked') != false){
              $(this).prop('checked', false);
              $(this).trigger('change');
            }
          });
        });
        $("input.dainput-embedded").on('keyup', daAdjustInputWidth);
        $("input.dainput-embedded").each(daAdjustInputWidth);
        // $(".dahelptrigger").click(function(e) {
        //   e.preventDefault();
        //   $(this).tab('show');
        // });
        //$("#daquestionlabel").click(function(e) {
        //  e.preventDefault();
        //  $(this).tab('show');
        //});
        $('#dapagetitle').click(function(e) {
          if ($(this).prop('href') == '#'){
            e.preventDefault();
            //$('#daquestionlabel').tab('show');
          }
        });
        $('select.damultiselect').each(function(){
          var varname = atob($(this).data('varname'));
          var theSelect = this;
          $(this).find('option').each(function(){
            var theVal = atob($(this).data('valname'));
            var key = varname + '["' + theVal + '"]';
            if (!daVarLookupSelect[key]){
              daVarLookupSelect[key] = [];
            }
            daVarLookupSelect[key].push({'select': theSelect, 'option': this});
            key = varname + "['" + theVal + "']"
            if (!daVarLookupSelect[key]){
              daVarLookupSelect[key] = [];
            }
            daVarLookupSelect[key].push({'select': theSelect, 'option': this});
          });
        })
        $('.dacurrency').each(function(){
          var theVal = $(this).val().toString();
          if (theVal.indexOf('.') >= 0 || theVal.indexOf(',') >= 0){
            var num = parseFloat(theVal);
            var cleanNum = num.toFixed(""" + str(daconfig.get('currency decimal places', 2)) + """);
            $(this).val(cleanNum);
          }
        });
        $('.dacurrency').on('blur', function(){
          var theVal = $(this).val().toString();
          if (theVal.indexOf('.') >= 0 || theVal.indexOf(',') >= 0){
            var num = parseFloat(theVal);
            var cleanNum = num.toFixed(""" + str(daconfig.get('currency decimal places', 2)) + """);
            if (cleanNum != 'NaN') {
              $(this).val(cleanNum);
            }
          }
        });
        $("#dahelp").on("shown.bs.tab", function(){
          window.scrollTo(0, 1);
          $("#dahelptoggle").removeClass('daactivetext')
          $("#dahelptoggle").blur();
        });
        $("#dasourcetoggle").on("click", function(){
          $(this).parent().toggleClass("active");
          $(this).blur();
        });
        $('#dabackToQuestion').click(function(event){
          $('#daquestionlabel').tab('show');
        });
        daShowIfInProcess = true;
        var daTriggerQueries = [];
        function daOnlyUnique(value, index, self){
          return self.indexOf(value) === index;
        }
        $(".dajsshowif").each(function(){
          var showIfDiv = this;
          var jsInfo = JSON.parse(atob($(this).data('jsshowif')));
          var showIfSign = jsInfo['sign'];
          var showIfMode = jsInfo['mode'];
          var jsExpression = jsInfo['expression'];
          var n = jsInfo['vars'].length;
          for (var i = 0; i < n; ++i){
            var showIfVars = [];
            var initShowIfVar = btoa(jsInfo['vars'][i]).replace(/[\\n=]/g, '');
            var initShowIfVarEscaped = initShowIfVar.replace(/(:|\.|\[|\]|,|=)/g, "\\\\$1");
            var elem = $("[name='" + initShowIfVarEscaped + "']");
            if (elem.length > 0){
              showIfVars.push(initShowIfVar);
            }
            if (daVarLookupMulti.hasOwnProperty(initShowIfVar)){
              for (var j = 0; j < daVarLookupMulti[initShowIfVar].length; j++){
                var altShowIfVar = daVarLookupMulti[initShowIfVar][j];
                var altShowIfVarEscaped = altShowIfVar.replace(/(:|\.|\[|\]|,|=)/g, "\\\\$1");
                var altElem = $("[name='" + altShowIfVarEscaped + "']");
                if (altElem.length > 0 && !$.contains(this, altElem[0])){
                  showIfVars.push(altShowIfVar);
                }
              }
            }
            if (showIfVars.length == 0){
              console.log("ERROR: reference to non-existent field " + jsInfo['vars'][i]);
            }
            for (var j = 0; j < showIfVars.length; ++j){
              var showIfVar = showIfVars[j];
              var showIfVarEscaped = showIfVar.replace(/(:|\.|\[|\]|,|=)/g, "\\\\$1");
              var showHideDiv = function(speed){
                var elem = daGetField(jsInfo['vars'][i]);
                if (elem != null && !$(elem).parents('.da-form-group').first().is($(this).parents('.da-form-group').first())){
                  return;
                }
                var resultt = eval(jsExpression);
                if(resultt){
                  if (showIfSign){
                    if ($(showIfDiv).data('isVisible') != '1'){
                      daShowHideHappened = true;
                    }
                    if (showIfMode == 0){
                      $(showIfDiv).show(speed);
                    }
                    $(showIfDiv).data('isVisible', '1');
                    $(showIfDiv).find('input, textarea, select').prop("disabled", false);
                    $(showIfDiv).find('input.combobox').each(function(){
                      daComboBoxes[$(this).attr('id')].enable();
                    });
                  }
                  else{
                    if ($(showIfDiv).data('isVisible') != '0'){
                      daShowHideHappened = true;
                    }
                    if (showIfMode == 0){
                      $(showIfDiv).hide(speed);
                    }
                    $(showIfDiv).data('isVisible', '0');
                    $(showIfDiv).find('input, textarea, select').prop("disabled", true);
                    $(showIfDiv).find('input.combobox').each(function(){
                      daComboBoxes[$(this).attr('id')].disable();
                    });
                  }
                }
                else{
                  if (showIfSign){
                    if ($(showIfDiv).data('isVisible') != '0'){
                      daShowHideHappened = true;
                    }
                    if (showIfMode == 0){
                      $(showIfDiv).hide(speed);
                    }
                    $(showIfDiv).data('isVisible', '0');
                    $(showIfDiv).find('input, textarea, select').prop("disabled", true);
                    $(showIfDiv).find('input.combobox').each(function(){
                      daComboBoxes[$(this).attr('id')].disable();
                    });
                  }
                  else{
                    if ($(showIfDiv).data('isVisible') != '1'){
                      daShowHideHappened = true;
                    }
                    if (showIfMode == 0){
                      $(showIfDiv).show(speed);
                    }
                    $(showIfDiv).data('isVisible', '1');
                    $(showIfDiv).find('input, textarea, select').prop("disabled", false);
                    $(showIfDiv).find('input.combobox').each(function(){
                      daComboBoxes[$(this).attr('id')].enable();
                    });
                  }
                }
                var daThis = this;
                if (!daShowIfInProcess){
                  daShowIfInProcess = true;
                  $(":input").not("[type='file']").each(function(){
                    if (this != daThis){
                      $(this).trigger('change');
                    }
                  });
                  daShowIfInProcess = false;
                }
              };
              var showHideDivImmediate = function(){
                showHideDiv.apply(this, [null]);
              }
              var showHideDivFast = function(){
                showHideDiv.apply(this, ['fast']);
              }
              daTriggerQueries.push("#" + showIfVarEscaped);
              daTriggerQueries.push("input[type='radio'][name='" + showIfVarEscaped + "']");
              daTriggerQueries.push("input[type='checkbox'][name='" + showIfVarEscaped + "']");
              $("#" + showIfVarEscaped).change(showHideDivFast);
              $("input[type='radio'][name='" + showIfVarEscaped + "']").change(showHideDivFast);
              $("input[type='checkbox'][name='" + showIfVarEscaped + "']").change(showHideDivFast);
              $("#" + showIfVarEscaped).on('daManualTrigger', showHideDivImmediate);
              $("input[type='radio'][name='" + showIfVarEscaped + "']").on('daManualTrigger', showHideDivImmediate);
              $("input[type='checkbox'][name='" + showIfVarEscaped + "']").on('daManualTrigger', showHideDivImmediate);
            }
          }
        });
        $(".dashowif").each(function(){
          var showIfVars = [];
          var showIfSign = $(this).data('showif-sign');
          var showIfMode = parseInt($(this).data('showif-mode'));
          var initShowIfVar = $(this).data('showif-var');
          var varName = atob(initShowIfVar);
          var initShowIfVarEscaped = initShowIfVar.replace(/(:|\.|\[|\]|,|=)/g, "\\\\$1");
          var elem = $("[name='" + initShowIfVarEscaped + "']");
          if (elem.length > 0){
            showIfVars.push(initShowIfVar);
          }
          if (daVarLookupMulti.hasOwnProperty(initShowIfVar)){
            var n = daVarLookupMulti[initShowIfVar].length;
            for (var i = 0; i < n; i++){
              var altShowIfVar = daVarLookupMulti[initShowIfVar][i];
              var altShowIfVarEscaped = altShowIfVar.replace(/(:|\.|\[|\]|,|=)/g, "\\\\$1");
              var altElem = $("[name='" + altShowIfVarEscaped + "']");
              if (altElem.length > 0 && !$.contains(this, altElem[0])){
                showIfVars.push(altShowIfVar);
              }
            }
          }
          var showIfVal = $(this).data('showif-val');
          var saveAs = $(this).data('saveas');
          var showIfDiv = this;
          var n = showIfVars.length;
          for (var i = 0; i < n; ++i){
            var showIfVar = showIfVars[i];
            var showIfVarEscaped = showIfVar.replace(/(:|\.|\[|\]|,|=)/g, "\\\\$1");
            var showHideDiv = function(speed){
              var elem = daGetField(varName, showIfDiv);
              if (elem != null && !$(elem).parents('.da-form-group').first().is($(this).parents('.da-form-group').first())){
                return;
              }
              var theVal;
              var showifParents = $(this).parents(".dashowif");
              if (showifParents.length !== 0 && !($(showifParents[0]).data("isVisible") == '1')){
                theVal = '';
                //console.log("Setting theVal to blank.");
              }
              else if ($(this).attr('type') == "checkbox"){
                theVal = $("input[name='" + showIfVarEscaped + "']:checked").val();
                if (typeof(theVal) == 'undefined'){
                  //console.log('manually setting checkbox value to False');
                  theVal = 'False';
                }
              }
              else if ($(this).attr('type') == "radio"){
                theVal = $("input[name='" + showIfVarEscaped + "']:checked").val();
                if (typeof(theVal) == 'undefined'){
                  theVal = '';
                }
                else if (theVal != '' && $("input[name='" + showIfVarEscaped + "']:checked").hasClass("daobject")){
                  try{
                    theVal = atob(theVal);
                  }
                  catch(e){
                  }
                }
              }
              else{
                theVal = $(this).val();
                if (theVal != '' && $(this).hasClass("daobject")){
                  try{
                    theVal = atob(theVal);
                  }
                  catch(e){
                  }
                }
              }
              //console.log("this is " + $(this).attr('id') + " and saveAs is " + atob(saveAs) + " and showIfVar is " + atob(showIfVar) + " and val is " + String(theVal) + " and showIfVal is " + String(showIfVal));
              if(daShowIfCompare(theVal, showIfVal)){
                if (showIfSign){
                  if ($(showIfDiv).data('isVisible') != '1'){
                    daShowHideHappened = true;
                  }
                  if (showIfMode == 0){
                    $(showIfDiv).show(speed);
                  }
                  $(showIfDiv).data('isVisible', '1');
                  $(showIfDiv).find('input, textarea, select').prop("disabled", false);
                  $(showIfDiv).find('input.combobox').each(function(){
                    daComboBoxes[$(this).attr('id')].enable();
                  });
                }
                else{
                  if ($(showIfDiv).data('isVisible') != '0'){
                    daShowHideHappened = true;
                  }
                  if (showIfMode == 0){
                    $(showIfDiv).hide(speed);
                  }
                  $(showIfDiv).data('isVisible', '0');
                  $(showIfDiv).find('input, textarea, select').prop("disabled", true);
                  $(showIfDiv).find('input.combobox').each(function(){
                    daComboBoxes[$(this).attr('id')].disable();
                  });
                }
              }
              else{
                if (showIfSign){
                  if ($(showIfDiv).data('isVisible') != '0'){
                    daShowHideHappened = true;
                  }
                  if (showIfMode == 0){
                    $(showIfDiv).hide(speed);
                  }
                  $(showIfDiv).data('isVisible', '0');
                  $(showIfDiv).find('input, textarea, select').prop("disabled", true);
                  $(showIfDiv).find('input.combobox').each(function(){
                    daComboBoxes[$(this).attr('id')].disable();
                  });
                }
                else{
                  if ($(showIfDiv).data('isVisible') != '1'){
                    daShowHideHappened = true;
                  }
                  if (showIfMode == 0){
                    $(showIfDiv).show(speed);
                  }
                  $(showIfDiv).data('isVisible', '1');
                  $(showIfDiv).find('input, textarea, select').prop("disabled", false);
                  $(showIfDiv).find('input.combobox').each(function(){
                    daComboBoxes[$(this).attr('id')].enable();
                  });
                }
              }
              var daThis = this;
              if (!daShowIfInProcess){
                daShowIfInProcess = true;
                $(":input").not("[type='file']").each(function(){
                  if (this != daThis){
                    $(this).trigger('change');
                  }
                });
                daShowIfInProcess = false;
              }
            };
            var showHideDivImmediate = function(){
              showHideDiv.apply(this, [null]);
            }
            var showHideDivFast = function(){
              showHideDiv.apply(this, ['fast']);
            }
            daTriggerQueries.push("#" + showIfVarEscaped);
            daTriggerQueries.push("input[type='radio'][name='" + showIfVarEscaped + "']");
            daTriggerQueries.push("input[type='checkbox'][name='" + showIfVarEscaped + "']");
            $("#" + showIfVarEscaped).change(showHideDivFast);
            $("#" + showIfVarEscaped).on('daManualTrigger', showHideDivImmediate);
            $("input[type='radio'][name='" + showIfVarEscaped + "']").change(showHideDivFast);
            $("input[type='radio'][name='" + showIfVarEscaped + "']").on('daManualTrigger', showHideDivImmediate);
            $("input[type='checkbox'][name='" + showIfVarEscaped + "']").change(showHideDivFast);
            $("input[type='checkbox'][name='" + showIfVarEscaped + "']").on('daManualTrigger', showHideDivImmediate);
          }
        });
        function daTriggerAllShowHides(){
          var daUniqueTriggerQueries = daTriggerQueries.filter(daOnlyUnique);
          var daFirstTime = true;
          var daTries = 0;
          while ((daFirstTime || daShowHideHappened) && ++daTries < 100){
            daShowHideHappened = false;
            daFirstTime = false;
            var n = daUniqueTriggerQueries.length;
            for (var i = 0; i < n; ++i){
              $(daUniqueTriggerQueries[i]).trigger('daManualTrigger');
            }
          }
          if (daTries >= 100){
            console.log("Too many contradictory 'show if' conditions");
          }
        }
        if (daTriggerQueries.length > 0){
          daTriggerAllShowHides();
        }
        $(".danavlink").last().addClass('thelast');
        $(".danavlink").each(function(){
          if ($(this).hasClass('btn') && !$(this).hasClass('danotavailableyet')){
            var the_a = $(this);
            var the_delay = 1000 + 250 * parseInt($(this).data('index'));
            setTimeout(function(){
              $(the_a).removeClass('""" + app.config['BUTTON_STYLE'] + """secondary');
              if ($(the_a).hasClass('active')){
                $(the_a).addClass('""" + app.config['BUTTON_STYLE'] + """success');
              }
              else{
                $(the_a).addClass('""" + app.config['BUTTON_STYLE'] + """warning');
              }
            }, the_delay);
          }
        });
        daShowIfInProcess = false;
        // daDisable = setTimeout(function(){
        //   $("#daform").find('button[type="submit"]').prop("disabled", true);
        //   //$("#daform").find(':input').prop("disabled", true);
        // }, 1);
        $("#daform").each(function(){
          $(this).find(':input').on('change', daPushChanges);
        });
        daInitialized = true;
        daShowingHelp = 0;
        setTimeout(function(){
          $("#daflash .alert-success").hide(300, function(){
            $(self).remove();
          });
        }, 3000);
        $(document).trigger('daPageLoad');
      }
      $( document ).ready(function(){
        daInitialize(1);
        $( window ).bind('unload', function() {
          if (daSocket != null && daSocket.connected){
            daSocket.emit('terminate');
          }
        });
        if (location.protocol === 'http:' || document.location.protocol === 'http:'){
            daSocket = io.connect('http://' + document.domain + '/observer', {path: '""" + ROOT + """ws/socket.io', query: "i=" + daYamlFilename});
        }
        if (location.protocol === 'https:' || document.location.protocol === 'https:'){
            daSocket = io.connect('https://' + document.domain + '/observer', {path: '""" + ROOT + """ws/socket.io', query: "i=" + daYamlFilename});
        }
        if (typeof daSocket !== 'undefined') {
            daSocket.on('connect', function() {
                //console.log("Connected!");
                daSocket.emit('observe', {uid: """ + json.dumps(uid) + """, i: daYamlFilename, userid: """ + json.dumps(str(userid)) + """});
                daConnected = true;
            });
            daSocket.on('terminate', function() {
                //console.log("Terminating socket");
                daSocket.disconnect();
            });
            daSocket.on('disconnect', function() {
                //console.log("Disconnected socket");
                //daSocket = null;
            });
            daSocket.on('stopcontrolling', function(data) {
                window.parent.daStopControlling(data.key);
            });
            daSocket.on('start_being_controlled', function(data) {
                //console.log("Got start_being_controlled");
                daConfirmed = true;
                daPushChanges();
                window.parent.daGotConfirmation(data.key);
            });
            daSocket.on('abortcontrolling', function(data) {
                //console.log("Got abortcontrolling");
                //daSendChanges = false;
                //daConfirmed = false;
                //daStopPushChanges();
                window.parent.daAbortControlling(data.key);
            });
            daSocket.on('noconnection', function(data) {
                //console.log("warning: no connection");
                if (daNoConnectionCount++ > 2){
                    //console.log("error: no connection");
                    window.parent.daStopControlling(data.key);
                }
            });
            daSocket.on('newpage', function(incoming) {
                //console.log("Got newpage")
                var data = incoming.obj;
                $(daTargetDiv).html(data.body);
                $(daTargetDiv).parent().removeClass();
                $(daTargetDiv).parent().addClass(data.bodyclass);
                daInitialize(1);
                var tempDiv = document.createElement('div');
                tempDiv.innerHTML = data.extra_scripts;
                var scripts = tempDiv.getElementsByTagName('script');
                for (var i = 0; i < scripts.length; i++){
                  if (scripts[i].src != ""){
                    daAddScriptToHead(scripts[i].src);
                  }
                  else{
                    daGlobalEval(scripts[i].innerHTML);
                  }
                }
                for (var i = 0; i < data.extra_css.length; i++){
                  $("head").append(data.extra_css[i]);
                }
                document.title = data.browser_title;
                if ($("html").attr("lang") != data.lang){
                  $("html").attr("lang", data.lang);
                }
                daPushChanges();
            });
            daSocket.on('pushchanges', function(data) {
                //console.log("Got pushchanges: " + JSON.stringify(data));
                var valArray = Object();
                var values = data.parameters;
                for (var i = 0; i < values.length; i++) {
                    valArray[values[i].name] = values[i].value;
                }
                $("#daform").each(function(){
                    $(this).find(':input').each(function(){
                        var type = $(this).attr('type');
                        var id = $(this).attr('id');
                        var name = $(this).attr('name');
                        if (type == 'checkbox'){
                            if (name in valArray){
                                if (valArray[name] == 'True'){
                                    if ($(this).prop('checked') != true){
                                        $(this).prop('checked', true);
                                        $(this).trigger('change');
                                    }
                                }
                                else{
                                    if ($(this).prop('checked') != false){
                                        $(this).prop('checked', false);
                                        $(this).trigger('change');
                                    }
                                }
                            }
                            else{
                                if ($(this).prop('checked') != false){
                                    $(this).prop('checked', false);
                                    $(this).trigger('change');
                                }
                            }
                        }
                        else if (type == 'radio'){
                            if (name in valArray){
                                if (valArray[name] == $(this).val()){
                                    if ($(this).prop('checked') != true){
                                        $(this).prop('checked', true);
                                        $(this).trigger('change');
                                    }
                                }
                                else{
                                    if ($(this).prop('checked') != false){
                                        $(this).prop('checked', false);
                                        $(this).trigger('change');
                                    }
                                }
                            }
                        }
                        else if ($(this).data().hasOwnProperty('sliderMax')){
                            $(this).slider('setValue', parseInt(valArray[name]));
                        }
                        else{
                            if (name in valArray){
                                $(this).val(valArray[name]);
                            }
                        }
                    });
                });
            });
        }
        daObserverChangesInterval = setInterval(daPushChanges, """ + str(CHECKIN_INTERVAL) + """);
    });
    </script>
"""
    the_key = 'da:html:uid:' + str(uid) + ':i:' + str(i) + ':userid:' + str(userid)
    html = r.get(the_key)
    if html is not None:
        obj = json.loads(html.decode())
    else:
        logmessage("observer: failed to load JSON from key " + the_key)
        obj = {}
    page_title = word('Observation')
    output = standard_html_start(interview_language=obj.get('lang', 'en'), debug=DEBUG, bootstrap_theme=obj.get('bootstrap_theme', None))
    output += obj.get('global_css', '') + "\n" + indent_by("".join(obj.get('extra_css', [])), 4)
    output += '\n    <title>' + page_title + '</title>\n  </head>\n  <body class="' + obj.get('bodyclass', 'dabody da-pad-for-navbar da-pad-for-footer') + '">\n  <div id="dabody">\n  '
    output += obj.get('body', '')
    output += "    </div>\n    </div>" + standard_scripts(interview_language=obj.get('lang', 'en')) + observation_script + "\n    " + "".join(obj.get('extra_scripts', [])) + "\n  </body>\n</html>"
    response = make_response(output.encode('utf-8'), '200 OK')
    response.headers['Content-type'] = 'text/html; charset=utf-8'
    return response

def decode_dict(the_dict):
    out_dict = {}
    for k, v in the_dict.items():
        out_dict[k.decode()] = v.decode()
    return out_dict

@app.route('/monitor', methods=['GET', 'POST'])
@login_required
@roles_required(['admin', 'advocate'])
def monitor():
    if not app.config['ENABLE_MONITOR']:
        return ('File not found', 404)
    setup_translation()
    if request.method == 'GET' and needs_to_change_password():
        return redirect(url_for('user.change_password', next=url_for('monitor')))
    session['monitor'] = 1
    if 'user_id' not in session:
        session['user_id'] = current_user.id
    phone_number_key = 'da:monitor:phonenumber:' + str(session['user_id'])
    default_phone_number = r.get(phone_number_key)
    if default_phone_number is None:
        default_phone_number = ''
    else:
        default_phone_number = default_phone_number.decode()
    sub_role_key = 'da:monitor:userrole:' + str(session['user_id'])
    if r.exists(sub_role_key):
        subscribed_roles = decode_dict(r.hgetall(sub_role_key))
        r.expire(sub_role_key, 2592000)
    else:
        subscribed_roles = {}
    key = 'da:monitor:available:' + str(current_user.id)
    if r.exists(key):
        daAvailableForChat = 'true'
    else:
        daAvailableForChat = 'false'
    call_forwarding_on = 'false'
    if twilio_config is not None:
        forwarding_phone_number = twilio_config['name']['default'].get('number', None)
        if forwarding_phone_number is not None:
            call_forwarding_on = 'true'
    script = "\n" + '    <script type="text/javascript" src="' + url_for('static', filename='app/socket.io.min.js', v=da_version) + '"></script>' + "\n" + """    <script type="text/javascript" charset="utf-8">
      var daAudioContext = null;
      var daSocket;
      var daSoundBuffer = Object();
      var daShowingNotif = false;
      var daUpdatedSessions = Object();
      var daUserid = """ + str(current_user.id) + """;
      var daPhoneOnMessage = """ + json.dumps(word("The user can call you.  Click to cancel.")) + """;
      var daPhoneOffMessage = """ + json.dumps(word("Click if you want the user to be able to call you.")) + """;
      var daSessions = Object();
      var daAvailRoles = Object();
      var daChatPartners = Object();
      var daPhonePartners = Object();
      var daNewPhonePartners = Object();
      var daTermPhonePartners = Object();
      var daUsePhone = """ + call_forwarding_on + """;
      var daSubscribedRoles = """ + json.dumps(subscribed_roles) + """;
      var daAvailableForChat = """ + daAvailableForChat + """;
      var daPhoneNumber = """ + json.dumps(default_phone_number) + """;
      var daFirstTime = 1;
      var daUpdateMonitorInterval = null;
      var daNotificationsEnabled = false;
      var daControlling = Object();
      var daBrowserTitle = """ + json.dumps(word('Monitor')) + """;
      window.daGotConfirmation = function(key){
          //console.log("Got confirmation in parent for key " + key);
          // daControlling[key] = 2;
          // var skey = key.replace(/(:|\.|\[|\]|,|=|\/)/g, '\\\\$1');
          // $("#listelement" + skey).find("a").each(function(){
          //     if ($(this).data('name') == "stopcontrolling"){
          //         $(this).removeClass('dainvisible');
          //         console.log("Found it");
          //     }
          // });
      }
      function daFaviconRegular(){
        var link = document.querySelector("link[rel*='shortcut icon'") || document.createElement('link');
        link.type = 'image/x-icon';
        link.rel = 'shortcut icon';
        link.href = '""" + url_for('favicon', nocache="1") + """';
        document.getElementsByTagName('head')[0].appendChild(link);
      }
      function daFaviconAlert(){
        var link = document.querySelector("link[rel*='shortcut icon'") || document.createElement('link');
        link.type = 'image/x-icon';
        link.rel = 'shortcut icon';
        link.href = '""" + url_for('static', filename='app/chat.ico', v=da_version) + """?nocache=1';
        document.getElementsByTagName('head')[0].appendChild(link);
      }
      function daTopMessage(message){
          var newDiv = document.createElement('div');
          $(newDiv).addClass("datop-alert col-xs-10 col-sm-7 col-md-6 col-lg-5 dacol-centered");
          $(newDiv).html(message)
          $(newDiv).css("display", "none");
          $(newDiv).appendTo($(daTargetDiv));
          $(newDiv).slideDown();
          setTimeout(function(){
            $(newDiv).slideUp(300, function(){
              $(newDiv).remove();
            });
          }, 2000);
      }
      window.daAbortControlling = function(key){
          daTopMessage(""" + json.dumps(word("That screen is already being controlled by another operator")) + """);
          daStopControlling(key);
      }
      window.daStopControlling = function(key){
          //console.log("Got daStopControlling in parent for key " + key);
          // if (daControlling.hasOwnProperty(key)){
          //   delete daControlling[key];
          // }
          var skey = key.replace(/(:|\.|\[|\]|,|=|\/)/g, '\\\\$1');
          $("#listelement" + skey).find("a").each(function(){
              if ($(this).data('name') == "stopcontrolling"){
                  $(this).click();
                  //console.log("Found it");
              }
          });
      }
      function daOnError(){
          console.log('daOnError');
      }
      function daLoadSoundBuffer(key, url_a, url_b){
          //console.log("daLoadSoundBuffer");
          var pos = 0;
          if (daAudioContext == null){
              return;
          }
          var request = new XMLHttpRequest();
          request.open('GET', url_a, true);
          request.responseType = 'arraybuffer';
          request.onload = function(){
              daAudioContext.decodeAudioData(request.response, function(buffer){
                  if (!buffer){
                      if (pos == 1){
                          console.error('daLoadSoundBuffer: error decoding file data');
                          return;
                      }
                      else {
                          pos = 1;
                          console.info('daLoadSoundBuffer: error decoding file data, trying next source');
                          request.open("GET", url_b, true);
                          return request.send();
                      }
                  }
                  daSoundBuffer[key] = buffer;
              },
              function(error){
                  if (pos == 1){
                      console.error('daLoadSoundBuffer: decodeAudioData error');
                      return;
                  }
                  else{
                      pos = 1;
                      console.info('daLoadSoundBuffer: decodeAudioData error, trying next source');
                      request.open("GET", url_b, true);
                      return request.send();
                  }
              });
          }
          request.send();
      }
      function daPlaySound(key) {
          //console.log("daPlaySound");
          var buffer = daSoundBuffer[key];
          if (!daAudioContext || !buffer){
              return;
          }
          var source = daAudioContext.createBufferSource();
          source.buffer = buffer;
          source.connect(daAudioContext.destination);
          source.start(0);
      }
      function daCheckNotifications(){
          //console.log("daCheckNotifications");
          if (daNotificationsEnabled){
              return;
          }
          if (!("Notification" in window)) {
              daNotificationsEnabled = false;
              return;
          }
          if (Notification.permission === "granted") {
              daNotificationsEnabled = true;
              return;
          }
          if (Notification.permission !== 'denied') {
              Notification.requestPermission(function (permission) {
                  if (permission === "granted") {
                      daNotificationsEnabled = true;
                  }
              });
          }
      }
      function daNotifyOperator(key, mode, message) {
          //console.log("daNotifyOperator: " + key + " " + mode + " " + message);
          var skey = key.replace(/(:|\.|\[|\]|,|=|\/)/g, '\\\\$1');
          if (mode == "chat"){
            daPlaySound('newmessage');
          }
          else{
            daPlaySound('newconversation');
          }
          if ($("#listelement" + skey).offset().top > $(window).scrollTop() + $(window).height()){
            if (mode == "chat"){
              $("#chat-message-below").html(""" + json.dumps(word("New message below")) + """);
            }
            else{
              $("#chat-message-below").html(""" + json.dumps(word("New conversation below")) + """);
            }
            //$("#chat-message-below").data('key', key);
            $("#chat-message-below").slideDown();
            daShowingNotif = true;
            daMarkAsUpdated(key);
          }
          else if ($("#listelement" + skey).offset().top + $("#listelement" + skey).height() < $(window).scrollTop() + 32){
            if (mode == "chat"){
              $("#chat-message-above").html(""" + json.dumps(word("New message above")) + """);
            }
            else{
              $("#chat-message-above").html(""" + json.dumps(word("New conversation above")) + """);
            }
            //$("#chat-message-above").data('key', key);
            $("#chat-message-above").slideDown();
            daShowingNotif = true;
            daMarkAsUpdated(key);
          }
          else{
            //console.log("It is visible");
          }
          if (!daNotificationsEnabled){
              //console.log("Browser will not enable notifications")
              return;
          }
          if (!("Notification" in window)) {
              return;
          }
          if (Notification.permission === "granted") {
              var notification = new Notification(message);
          }
          else if (Notification.permission !== 'denied') {
              Notification.requestPermission(function (permission) {
                  if (permission === "granted") {
                      var notification = new Notification(message);
                      daNotificationsEnabled = true;
                  }
              });
          }
      }
      function daPhoneNumberOk(){
          //console.log("daPhoneNumberOk");
          var phoneNumber = $("#daPhoneNumber").val();
          if (phoneNumber == '' || phoneNumber.match(/^\+?[1-9]\d{1,14}$/)){
              return true;
          }
          else{
              return false;
          }
      }
      function daCheckPhone(){
          //console.log("daCheckPhone");
          $("#daPhoneNumber").val($("#daPhoneNumber").val().replace(/ \-/g, ''));
          var the_number = $("#daPhoneNumber").val();
          if (the_number != '' && the_number[0] != '+'){
              $("#daPhoneNumber").val('+' + the_number);
          }
          if (daPhoneNumberOk()){
              $("#daPhoneNumber").removeClass("is-invalid");
              $("#daPhoneError").addClass("dainvisible");
              daPhoneNumber = $("#daPhoneNumber").val();
              if (daPhoneNumber == ''){
                  daPhoneNumber = null;
              }
              else{
                  $(".phone").removeClass("dainvisible");
              }
              $("#daPhoneSaved").removeClass("dainvisible");
              setTimeout(function(){
                  $("#daPhoneSaved").addClass("dainvisible");
              }, 2000);
          }
          else{
              $("#daPhoneNumber").addClass("is-invalid");
              $("#daPhoneError").removeClass("dainvisible");
              daPhoneNumber = null;
              $(".phone").addClass("dainvisible");
          }
      }
      function daAllSessions(uid, yaml_filename){
          //console.log("daAllSessions");
          var prefix = 'da:session:uid:' + uid + ':i:' + yaml_filename + ':userid:';
          var output = Array();
          for (var key in daSessions){
              if (daSessions.hasOwnProperty(key) && key.indexOf(prefix) == 0){
                  output.push(key);
              }
          }
          return(output);
      }
      function daScrollChat(key){
          var chatScroller = $(key).find('ul').first();
          if (chatScroller.length){
              var height = chatScroller[0].scrollHeight;
              chatScroller.animate({scrollTop: height}, 800);
          }
          else{
              console.log("daScrollChat: error")
          }
      }
      function daScrollChatFast(key){
          var chatScroller = $(key).find('ul').first();
          if (chatScroller.length){
            var height = chatScroller[0].scrollHeight;
              //console.log("Scrolling to " + height + " where there are " + chatScroller[0].childElementCount + " children");
              chatScroller.scrollTop(height);
            }
          else{
              console.log("daScrollChatFast: error")
          }
      }
      function daDoUpdateMonitor(){
          //console.log("daDoUpdateMonitor with " + daAvailableForChat);
          if (daPhoneNumberOk()){
            daPhoneNumber = $("#daPhoneNumber").val();
            if (daPhoneNumber == ''){
              daPhoneNumber = null;
            }
          }
          else{
            daPhoneNumber = null;
          }
          daSocket.emit('updatemonitor', {available_for_chat: daAvailableForChat, phone_number: daPhoneNumber, subscribed_roles: daSubscribedRoles, phone_partners_to_add: daNewPhonePartners, phone_partners_to_terminate: daTermPhonePartners});
      }
      function daUpdateMonitor(){
          //console.log("daUpdateMonitor with " + daAvailableForChat);
          if (daUpdateMonitorInterval != null){
              clearInterval(daUpdateMonitorInterval);
          }
          daDoUpdateMonitor();
          daUpdateMonitorInterval = setInterval(daDoUpdateMonitor, """ + str(CHECKIN_INTERVAL) + """);
          //console.log("daUpdateMonitor");
      }
      function daIsHidden(ref){
          if ($(ref).length){
              if (($(ref).offset().top + $(ref).height() < $(window).scrollTop() + 32)){
                  return -1;
              }
              else if ($(ref).offset().top > $(window).scrollTop() + $(window).height()){
                  return 1;
              }
              else{
                  return 0;
              }
          }
          else{
              return 0;
          }
      }
      function daMarkAsUpdated(key){
          //console.log("daMarkAsUpdated with " + key);
          var skey = key.replace(/(:|\.|\[|\]|,|=|\/)/g, '\\\\$1');
          if (daIsHidden("#listelement" + skey)){
              daUpdatedSessions["#listelement" + skey] = 1;
          }
      }
      function daActivateChatArea(key){
          //console.log("daActivateChatArea with " + key);
          var skey = key.replace(/(:|\.|\[|\]|,|=|\/)/g, '\\\\$1');
          if (!$("#chatarea" + skey).find('input').first().is(':focus')){
            $("#listelement" + skey).addClass("da-new-message");
            if (daBrowserTitle == document.title){
              document.title = '* ' + daBrowserTitle;
              daFaviconAlert();
            }
          }
          daMarkAsUpdated(key);
          $("#chatarea" + skey).removeClass('dainvisible');
          $("#chatarea" + skey).find('input, button').prop("disabled", false);
          $("#chatarea" + skey).find('ul').html('');
          daSocket.emit('chat_log', {key: key});
      }
      function daDeActivateChatArea(key){
          //console.log("daActivateChatArea with " + key);
          var skey = key.replace(/(:|\.|\[|\]|,|=|\/)/g, '\\\\$1');
          $("#chatarea" + skey).find('input, button').prop("disabled", true);
          $("#listelement" + skey).removeClass("da-new-message");
          if (document.title != daBrowserTitle){
              document.title = daBrowserTitle;
              daFaviconRegular();
          }
      }
      function daUndrawSession(key){
          //console.log("Undrawing...");
          var skey = key.replace(/(:|\.|\[|\]|,|=|\/)/g, '\\\\$1');
          var xButton = document.createElement('a');
          var xButtonIcon = document.createElement('i');
          $(xButton).addClass("dacorner-remove");
          $(xButtonIcon).addClass("fas fa-times-circle");
          $(xButtonIcon).appendTo($(xButton));
          $("#listelement" + skey).addClass("list-group-item-danger");
          $("#session" + skey).find("a").remove();
          $("#session" + skey).find("span").first().html(""" + json.dumps(word("offline")) + """);
          $("#session" + skey).find("span").first().removeClass('""" + app.config['BUTTON_STYLE'] + """info');
          $("#session" + skey).find("span").first().addClass('""" + app.config['BUTTON_STYLE'] + """danger');
          $(xButton).click(function(){
              $("#listelement" + skey).slideUp(300, function(){
                  $("#listelement" + skey).remove();
                  daCheckIfEmpty();
              });
          });
          $(xButton).appendTo($("#session" + skey));
          $("#chatarea" + skey).find('input, button').prop("disabled", true);
          var theIframe = $("#iframe" + skey).find('iframe')[0];
          if (theIframe){
              $(theIframe).contents().find('body').addClass("dainactive");
              if (theIframe.contentWindow && theIframe.contentWindow.daTurnOffControl){
                  theIframe.contentWindow.daTurnOffControl();
              }
          }
          if (daControlling.hasOwnProperty(key)){
              delete daControlling[key];
          }
          delete daSessions[key];
      }
      function daPublishChatLog(uid, yaml_filename, userid, mode, messages, scroll){
          //console.log("daPublishChatLog with " + uid + " " + yaml_filename + " " + userid + " " + mode + " " + messages);
          //console.log("daPublishChatLog: scroll is " + scroll);
          var keys;
          //if (mode == 'peer' || mode == 'peerhelp'){
          //    keys = daAllSessions(uid, yaml_filename);
          //}
          //else{
              keys = ['da:session:uid:' + uid + ':i:' + yaml_filename + ':userid:' + userid];
          //}
          for (var i = 0; i < keys.length; ++i){
              key = keys[i];
              var skey = key.replace(/(:|\.|\[|\]|,|=|\/)/g, '\\\\$1');
              var chatArea = $("#chatarea" + skey).find('ul').first();
              if (messages.length > 0){
                $(chatArea).removeClass('dainvisible');
              }
              for (var i = 0; i < messages.length; ++i){
                  var message = messages[i];
                  var newLi = document.createElement('li');
                  $(newLi).addClass("list-group-item");
                  if (message.is_self){
                      $(newLi).addClass("list-group-item-primary dalistright");
                  }
                  else{
                      $(newLi).addClass("list-group-item-secondary dalistleft");
                  }
                  $(newLi).html(message.message);
                  $(newLi).appendTo(chatArea);
              }
              if (messages.length > 0 && scroll){
                  daScrollChatFast("#chatarea" + skey);
              }
          }
      }
      function daCheckIfEmpty(){
          if ($("#monitorsessions").find("li").length > 0){
              $("#emptylist").addClass("dainvisible");
          }
          else{
              $("#emptylist").removeClass("dainvisible");
          }
      }
      function daDrawSession(key, obj){
          //console.log("daDrawSession with " + key);
          var skey = key.replace(/(:|\.|\[|\]|,|=|\/)/g, '\\\\$1');
          var the_html;
          var wants_to_chat;
          if (obj.chatstatus != 'off'){ //obj.chatstatus == 'waiting' || obj.chatstatus == 'standby' || obj.chatstatus == 'ringing' || obj.chatstatus == 'ready' || obj.chatstatus == 'on' || obj.chatstatus == 'observeonly'
              wants_to_chat = true;
          }
          if (wants_to_chat){
              the_html = obj.browser_title + ' &mdash; '
              if (obj.hasOwnProperty('first_name')){
                the_html += obj.first_name + ' ' + obj.last_name;
              }
              else{
                the_html += """ + json.dumps(word("anonymous visitor") + ' ') + """ + obj.temp_user_id;
              }
          }
          var theListElement;
          var sessionDiv;
          var theIframeContainer;
          var theChatArea;
          if ($("#session" + skey).length && !(key in daSessions)){
              $("#listelement" + skey).removeClass("list-group-item-danger");
              $("#iframe" + skey).find('iframe').first().contents().find('body').removeClass("dainactive");
          }
          daSessions[key] = 1;
          if ($("#session" + skey).length){
              theListElement = $("#listelement" + skey).first();
              sessionDiv = $("#session" + skey).first();
              //controlDiv = $("#control" + skey).first();
              theIframeContainer = $("#iframe" + skey).first();
              theChatArea = $("#chatarea" + skey).first();
              $(sessionDiv).empty();
              if (obj.chatstatus == 'on' && key in daChatPartners && $("#chatarea" + skey).find('button').first().prop("disabled") == true){
                  daActivateChatArea(key);
              }
          }
          else{
              var theListElement = document.createElement('li');
              $(theListElement).addClass('list-group-item');
              $(theListElement).attr('id', "listelement" + key);
              var sessionDiv = document.createElement('div');
              $(sessionDiv).attr('id', "session" + key);
              $(sessionDiv).addClass('da-chat-session');
              $(sessionDiv).addClass('p-1');
              $(sessionDiv).appendTo($(theListElement));
              $(theListElement).appendTo("#monitorsessions");
              // controlDiv = document.createElement('div');
              // $(controlDiv).attr('id', "control" + key);
              // $(controlDiv).addClass("dachatcontrol dainvisible da-chat-session");
              // $(controlDiv).appendTo($(theListElement));
              theIframeContainer = document.createElement('div');
              $(theIframeContainer).addClass("daobserver-container dainvisible");
              $(theIframeContainer).attr('id', 'iframe' + key);
              var theIframe = document.createElement('iframe');
              $(theIframe).addClass("daobserver");
              $(theIframe).attr('name', 'iframe' + key);
              $(theIframe).appendTo($(theIframeContainer));
              $(theIframeContainer).appendTo($(theListElement));
              var theChatArea = document.createElement('div');
              $(theChatArea).addClass('monitor-chat-area dainvisible');
              $(theChatArea).html('<div class="row"><div class="col-md-12"><ul class="list-group dachatbox" id="daCorrespondence"><\/ul><\/div><\/div><form autocomplete="off"><div class="row"><div class="col-md-12"><div class="input-group"><input type="text" class="form-control daChatMessage" disabled=""><button role="button" class="btn """ + app.config['BUTTON_STYLE'] + """secondary daChatButton" type="button" disabled="">""" + word("Send") + """<\/button><\/div><\/div><\/div><\/form>');
              $(theChatArea).attr('id', 'chatarea' + key);
              var submitter = function(){
                  //console.log("I am the submitter and I am submitting " + key);
                  var input = $(theChatArea).find("input").first();
                  var message = input.val().trim();
                  if (message == null || message == ""){
                      //console.log("Message was blank");
                      return false;
                  }
                  daSocket.emit('chatmessage', {key: key, data: input.val()});
                  input.val('');
                  return false;
              };
              $(theChatArea).find("button").click(submitter);
              $(theChatArea).find("input").bind('keypress keydown keyup', function(e){
                  var theCode = e.which || e.keyCode;
                  if(theCode == 13) { submitter(); e.preventDefault(); }
              });
              $(theChatArea).find("input").focus(function(){
                  $(theListElement).removeClass("da-new-message");
                  if (document.title != daBrowserTitle){
                      document.title = daBrowserTitle;
                      daFaviconRegular();
                  }
              });
              $(theChatArea).appendTo($(theListElement));
              if (obj.chatstatus == 'on' && key in daChatPartners){
                  daActivateChatArea(key);
              }
          }
          var theText = document.createElement('span');
          $(theText).addClass('da-chat-title-label');
          theText.innerHTML = the_html;
          var statusLabel = document.createElement('span');
          $(statusLabel).addClass("badge bg-info da-chat-status-label");
          $(statusLabel).html(obj.chatstatus == 'observeonly' ? 'off' : obj.chatstatus);
          $(statusLabel).appendTo($(sessionDiv));
          if (daUsePhone){
            var phoneButton = document.createElement('a');
            var phoneIcon = document.createElement('i');
            $(phoneIcon).addClass("fas fa-phone");
            $(phoneIcon).appendTo($(phoneButton));
            $(phoneButton).addClass("btn phone");
            $(phoneButton).data('name', 'phone');
            if (key in daPhonePartners){
              $(phoneButton).addClass("phone-on """ + app.config['BUTTON_STYLE'] + """success");
              $(phoneButton).attr('title', daPhoneOnMessage);
            }
            else{
              $(phoneButton).addClass("phone-off """ + app.config['BUTTON_STYLE'] + """secondary");
              $(phoneButton).attr('title', daPhoneOffMessage);
            }
            $(phoneButton).attr('tabindex', 0);
            $(phoneButton).addClass('daobservebutton')
            $(phoneButton).appendTo($(sessionDiv));
            $(phoneButton).attr('href', '#');
            if (daPhoneNumber == null){
              $(phoneButton).addClass("dainvisible");
            }
            $(phoneButton).click(function(e){
              e.preventDefault();
              if ($(this).hasClass("phone-off") && daPhoneNumber != null){
                $(this).removeClass("phone-off");
                $(this).removeClass(""" + '"' + app.config['BUTTON_STYLE'] + """secondary");
                $(this).addClass("phone-on");
                $(this).addClass(""" + '"' + app.config['BUTTON_STYLE'] + """success");
                $(this).attr('title', daPhoneOnMessage);
                daPhonePartners[key] = 1;
                daNewPhonePartners[key] = 1;
                if (key in daTermPhonePartners){
                  delete daTermPhonePartners[key];
                }
              }
              else{
                $(this).removeClass("phone-on");
                $(this).removeClass(""" + '"' + app.config['BUTTON_STYLE'] + """success");
                $(this).addClass("phone-off");
                $(this).addClass(""" + '"' + app.config['BUTTON_STYLE'] + """secondary");
                $(this).attr('title', daPhoneOffMessage);
                if (key in daPhonePartners){
                  delete daPhonePartners[key];
                }
                if (key in daNewPhonePartners){
                  delete daNewPhonePartners[key];
                }
                daTermPhonePartners[key] = 1;
              }
              daUpdateMonitor();
              return false;
            });
          }
          var unblockButton = document.createElement('a');
          $(unblockButton).addClass("btn """ + app.config['BUTTON_STYLE'] + """info daobservebutton");
          $(unblockButton).data('name', 'unblock');
          if (!obj.blocked){
              $(unblockButton).addClass("dainvisible");
          }
          $(unblockButton).html(""" + json.dumps(word("Unblock")) + """);
          $(unblockButton).attr('href', '#');
          $(unblockButton).appendTo($(sessionDiv));
          var blockButton = document.createElement('a');
          $(blockButton).addClass("btn """ + app.config['BUTTON_STYLE'] + """danger daobservebutton");
          if (obj.blocked){
              $(blockButton).addClass("dainvisible");
          }
          $(blockButton).html(""" + json.dumps(word("Block")) + """);
          $(blockButton).attr('href', '#');
          $(blockButton).data('name', 'block');
          $(blockButton).appendTo($(sessionDiv));
          $(blockButton).click(function(e){
              $(unblockButton).removeClass("dainvisible");
              $(this).addClass("dainvisible");
              daDeActivateChatArea(key);
              daSocket.emit('block', {key: key});
              e.preventDefault();
              return false;
          });
          $(unblockButton).click(function(e){
              $(blockButton).removeClass("dainvisible");
              $(this).addClass("dainvisible");
              daSocket.emit('unblock', {key: key});
              e.preventDefault();
              return false;
          });
          var joinButton = document.createElement('a');
          $(joinButton).addClass("btn """ + app.config['BUTTON_STYLE'] + """warning daobservebutton");
          $(joinButton).html(""" + json.dumps(word("Join")) + """);
          $(joinButton).attr('href', """ + json.dumps(url_for('visit_interview') + '?') + """ + $.param({i: obj.i, uid: obj.uid, userid: obj.userid}));
          $(joinButton).data('name', 'join');
          $(joinButton).attr('target', '_blank');
          $(joinButton).appendTo($(sessionDiv));
          if (wants_to_chat){
              var openButton = document.createElement('a');
              $(openButton).addClass("btn """ + app.config['BUTTON_STYLE'] + """primary daobservebutton");
              $(openButton).attr('href', """ + json.dumps(url_for('observer') + '?') + """ + $.param({i: obj.i, uid: obj.uid, userid: obj.userid}));
              //$(openButton).attr('href', 'about:blank');
              $(openButton).attr('id', 'observe' + key);
              $(openButton).attr('target', 'iframe' + key);
              $(openButton).html(""" + json.dumps(word("Observe")) + """);
              $(openButton).data('name', 'open');
              $(openButton).appendTo($(sessionDiv));
              var stopObservingButton = document.createElement('a');
              $(stopObservingButton).addClass("btn """ + app.config['BUTTON_STYLE'] + """secondary daobservebutton dainvisible");
              $(stopObservingButton).html(""" + json.dumps(word("Stop Observing")) + """);
              $(stopObservingButton).attr('href', '#');
              $(stopObservingButton).data('name', 'stopObserving');
              $(stopObservingButton).appendTo($(sessionDiv));
              var controlButton = document.createElement('a');
              $(controlButton).addClass("btn """ + app.config['BUTTON_STYLE'] + """info daobservebutton");
              $(controlButton).html(""" + json.dumps(word("Control")) + """);
              $(controlButton).attr('href', '#');
              $(controlButton).data('name', 'control');
              $(controlButton).appendTo($(sessionDiv));
              var stopControllingButton = document.createElement('a');
              $(stopControllingButton).addClass("btn """ + app.config['BUTTON_STYLE'] + """secondary daobservebutton dainvisible");
              $(stopControllingButton).html(""" + json.dumps(word("Stop Controlling")) + """);
              $(stopControllingButton).attr('href', '#');
              $(stopControllingButton).data('name', 'stopcontrolling');
              $(stopControllingButton).appendTo($(sessionDiv));
              $(controlButton).click(function(event){
                  event.preventDefault();
                  //console.log("Controlling...");
                  $(this).addClass("dainvisible");
                  $(stopControllingButton).removeClass("dainvisible");
                  $(stopObservingButton).addClass("dainvisible");
                  var theIframe = $("#iframe" + skey).find('iframe')[0];
                  if (theIframe != null && theIframe.contentWindow){
                      theIframe.contentWindow.daTurnOnControl();
                  }
                  else{
                      console.log("Cannot turn on control");
                  }
                  daControlling[key] = 1;
                  return false;
              });
              $(stopControllingButton).click(function(event){
                  //console.log("Got click on stopControllingButton");
                  event.preventDefault();
                  var theIframe = $("#iframe" + skey).find('iframe')[0];
                  if (theIframe != null && theIframe.contentWindow && theIframe.contentWindow.daTurnOffControl){
                      theIframe.contentWindow.daTurnOffControl();
                  }
                  else{
                      console.log("Cannot turn off control");
                      return false;
                  }
                  //console.log("Stop controlling...");
                  $(this).addClass("dainvisible");
                  $(controlButton).removeClass("dainvisible");
                  $(stopObservingButton).removeClass("dainvisible");
                  if (daControlling.hasOwnProperty(key)){
                      delete daControlling[key];
                  }
                  return false;
              });
              $(openButton).click(function(event){
                  //console.log("Observing..");
                  $(this).addClass("dainvisible");
                  $(stopObservingButton).removeClass("dainvisible");
                  $("#iframe" + skey).removeClass("dainvisible");
                  $(controlButton).removeClass("dainvisible");
                  return true;
              });
              $(stopObservingButton).click(function(e){
                  //console.log("Unobserving...");
                  $(this).addClass("dainvisible");
                  $(openButton).removeClass("dainvisible");
                  $(controlButton).addClass("dainvisible");
                  $(stopObservingButton).addClass("dainvisible");
                  $(stopControllingButton).addClass("dainvisible");
                  var theIframe = $("#iframe" + skey).find('iframe')[0];
                  if (daControlling.hasOwnProperty(key)){
                      delete daControlling[key];
                      if (theIframe != null && theIframe.contentWindow && theIframe.contentWindow.daTurnOffControl){
                          //console.log("Calling daTurnOffControl in iframe");
                          theIframe.contentWindow.daTurnOffControl();
                      }
                  }
                  if (theIframe != null && theIframe.contentWindow){
                      //console.log("Deleting the iframe");
                      theIframe.contentWindow.document.open();
                      theIframe.contentWindow.document.write("");
                      theIframe.contentWindow.document.close();
                  }
                  $("#iframe" + skey).slideUp(400, function(){
                      $(this).css("display", "").addClass("dainvisible");
                  });
                  e.preventDefault();
                  return false;
              });
              if ($(theIframeContainer).hasClass("dainvisible")){
                  $(openButton).removeClass("dainvisible");
                  $(stopObservingButton).addClass("dainvisible");
                  $(controlButton).addClass("dainvisible");
                  $(stopControllingButton).addClass("dainvisible");
                  if (daControlling.hasOwnProperty(key)){
                      delete daControlling[key];
                  }
              }
              else{
                  $(openButton).addClass("dainvisible");
                  if (daControlling.hasOwnProperty(key)){
                      $(stopObservingButton).addClass("dainvisible");
                      $(controlButton).addClass("dainvisible");
                      $(stopControllingButton).removeClass("dainvisible");
                  }
                  else{
                      $(stopObservingButton).removeClass("dainvisible");
                      $(controlButton).removeClass("dainvisible");
                      $(stopControllingButton).addClass("dainvisible");
                  }
              }
          }
          $(theText).appendTo($(sessionDiv));
          if (obj.chatstatus == 'on' && key in daChatPartners && $("#chatarea" + skey).hasClass('dainvisible')){
              daActivateChatArea(key);
          }
          if ((obj.chatstatus != 'on' || !(key in daChatPartners)) && $("#chatarea" + skey).find('button').first().prop("disabled") == false){
              daDeActivateChatArea(key);
          }
          else if (obj.blocked){
              daDeActivateChatArea(key);
          }
      }
      function daOnScrollResize(){
          if (document.title != daBrowserTitle){
              document.title = daBrowserTitle;
              daFaviconRegular();
          }
          if (!daShowingNotif){
              return true;
          }
          var obj = Array();
          for (var key in daUpdatedSessions){
              if (daUpdatedSessions.hasOwnProperty(key)){
                  obj.push(key);
              }
          }
          var somethingAbove = false;
          var somethingBelow = false;
          var firstElement = -1;
          var lastElement = -1;
          for (var i = 0; i < obj.length; ++i){
              var result = daIsHidden(obj[i]);
              if (result == 0){
                  delete daUpdatedSessions[obj[i]];
              }
              else if (result < 0){
                  var top = $(obj[i]).offset().top;
                  somethingAbove = true;
                  if (firstElement == -1 || top < firstElement){
                      firstElement = top;
                  }
              }
              else if (result > 0){
                  var top = $(obj[i]).offset().top;
                  somethingBelow = true;
                  if (lastElement == -1 || top > lastElement){
                      lastElement = top;
                  }
              }
          }
          if (($("#chat-message-above").is(":visible")) && !somethingAbove){
              $("#chat-message-above").hide();
          }
          if (($("#chat-message-below").is(":visible")) && !somethingBelow){
              $("#chat-message-below").hide();
          }
          if (!(somethingAbove || somethingBelow)){
              daShowingNotif = false;
          }
          return true;
      }
      $(document).ready(function(){
          //console.log("document ready");
          try {
              window.AudioContext = window.AudioContext || window.webkitAudioContext;
              daAudioContext = new AudioContext();
          }
          catch(e) {
              console.log('Web Audio API is not supported in this browser');
          }
          daLoadSoundBuffer('newmessage', '""" + url_for('static', filename='sounds/notification-click-on.mp3', v=da_version) + """', '""" + url_for('static', filename='sounds/notification-click-on.ogg', v=da_version) + """');
          daLoadSoundBuffer('newconversation', '""" + url_for('static', filename='sounds/notification-stapler.mp3', v=da_version) + """', '""" + url_for('static', filename='sounds/notification-stapler.ogg', v=da_version) + """');
          daLoadSoundBuffer('signinout', '""" + url_for('static', filename='sounds/notification-snap.mp3', v=da_version) + """', '""" + url_for('static', filename='sounds/notification-snap.ogg', v=da_version) + """');
          if (location.protocol === 'http:' || document.location.protocol === 'http:'){
              daSocket = io.connect('http://' + document.domain + '/monitor', {path: '""" + ROOT + """ws/socket.io'});
          }
          if (location.protocol === 'https:' || document.location.protocol === 'https:'){
              daSocket = io.connect('https://' + document.domain + '/monitor', {path: '""" + ROOT + """ws/socket.io'});
          }
          //console.log("socket is " + daSocket)
          if (typeof daSocket !== 'undefined') {
              daSocket.on('connect', function() {
                  //console.log("Connected!");
                  daUpdateMonitor();
              });
              daSocket.on('terminate', function() {
                  console.log("monitor: terminating socket");
                  daSocket.disconnect();
              });
              daSocket.on('disconnect', function() {
                  //console.log("monitor: disconnected socket");
                  //daSocket = null;
              });
              daSocket.on('refreshsessions', function(data) {
                  daUpdateMonitor();
              });
              // daSocket.on('abortcontroller', function(data) {
              //     console.log("Got abortcontroller message for " + data.key);
              // });
              daSocket.on('chatready', function(data) {
                  var key = 'da:session:uid:' + data.uid + ':i:' + data.i + ':userid:' + data.userid
                  //console.log('chatready: ' + key);
                  daActivateChatArea(key);
                  daNotifyOperator(key, "chatready", """ + json.dumps(word("New chat connection from")) + """ + ' ' + data.name)
              });
              daSocket.on('chatstop', function(data) {
                  var key = 'da:session:uid:' + data.uid + ':i:' + data.i + ':userid:' + data.userid
                  //console.log('chatstop: ' + key);
                  if (key in daChatPartners){
                      delete daChatPartners[key];
                  }
                  daDeActivateChatArea(key);
              });
              daSocket.on('chat_log', function(arg) {
                  //console.log('chat_log: ' + arg.userid);
                  daPublishChatLog(arg.uid, arg.i, arg.userid, arg.mode, arg.data, arg.scroll);
              });
              daSocket.on('block', function(arg) {
                  //console.log("back from blocking " + arg.key);
                  daUpdateMonitor();
              });
              daSocket.on('unblock', function(arg) {
                  //console.log("back from unblocking " + arg.key);
                  daUpdateMonitor();
              });
              daSocket.on('chatmessage', function(data) {
                  //console.log("chatmessage");
                  var keys;
                  if (data.data.mode == 'peer' || data.data.mode == 'peerhelp'){
                    keys = daAllSessions(data.uid, data.i);
                  }
                  else{
                    keys = ['da:session:uid:' + data.uid + ':i:' + data.i + ':userid:' + data.userid];
                  }
                  for (var i = 0; i < keys.length; ++i){
                    key = keys[i];
                    var skey = key.replace(/(:|\.|\[|\]|,|=|\/)/g, '\\\\$1');
                    //console.log("Received chat message for #chatarea" + skey);
                    var chatArea = $("#chatarea" + skey).find('ul').first();
                    var newLi = document.createElement('li');
                    $(newLi).addClass("list-group-item");
                    if (data.data.is_self){
                      $(newLi).addClass("list-group-item-primary dalistright");
                    }
                    else{
                      $(newLi).addClass("list-group-item-secondary dalistleft");
                    }
                    $(newLi).html(data.data.message);
                    $(newLi).appendTo(chatArea);
                    daScrollChat("#chatarea" + skey);
                    if (data.data.is_self){
                      $("#listelement" + skey).removeClass("da-new-message");
                      if (document.title != daBrowserTitle){
                        document.title = daBrowserTitle;
                        daFaviconRegular();
                      }
                    }
                    else{
                      if (!$("#chatarea" + skey).find('input').first().is(':focus')){
                        $("#listelement" + skey).addClass("da-new-message");
                        if (daBrowserTitle == document.title){
                          document.title = '* ' + daBrowserTitle;
                          daFaviconAlert();
                        }
                      }
                      if (data.data.hasOwnProperty('temp_user_id')){
                        daNotifyOperator(key, "chat", """ + json.dumps(word("anonymous visitor")) + """ + ' ' + data.data.temp_user_id + ': ' + data.data.message);
                      }
                      else{
                        if (data.data.first_name && data.data.first_name != ''){
                          daNotifyOperator(key, "chat", data.data.first_name + ' ' + data.data.last_name + ': ' + data.data.message);
                        }
                        else{
                          daNotifyOperator(key, "chat", data.data.email + ': ' + data.data.message);
                        }
                      }
                    }
                  }
              });
              daSocket.on('sessionupdate', function(data) {
                  //console.log("Got session update: " + data.session.chatstatus);
                  daDrawSession(data.key, data.session);
                  daCheckIfEmpty();
              });
              daSocket.on('updatemonitor', function(data) {
                  //console.log("Got update monitor response");
                  //console.log("updatemonitor: chat partners are: " + data.chatPartners);
                  daChatPartners = data.chatPartners;
                  daNewPhonePartners = Object();
                  daTermPhonePartners = Object();
                  daPhonePartners = data.phonePartners;
                  var newSubscribedRoles = Object();
                  for (var key in data.subscribedRoles){
                      if (data.subscribedRoles.hasOwnProperty(key)){
                          newSubscribedRoles[key] = 1;
                      }
                  }
                  for (var i = 0; i < data.availRoles.length; ++i){
                      var key = data.availRoles[i];
                      var skey = key.replace(/(:|\.|\[|\]|,|=|\/| )/g, '\\\\$1');
                      if ($("#role" + skey).length == 0){
                          var div = document.createElement('div');
                          $(div).addClass("form-check form-check-inline");
                          var label = document.createElement('label');
                          $(label).addClass('form-check-label');
                          $(label).attr('for', "role" + key);
                          var input = document.createElement('input');
                          $(input).addClass('form-check-input');
                          var text = document.createTextNode(key);
                          $(input).attr('type', 'checkbox');
                          $(input).attr('id', "role" + key);
                          if (key in newSubscribedRoles){
                              $(input).prop('checked', true);
                          }
                          else{
                              $(input).prop('checked', false);
                          }
                          $(input).val(key);
                          $(text).appendTo($(label));
                          $(input).appendTo($(div));
                          $(label).appendTo($(div));
                          $(div).appendTo($("#monitorroles"));
                          $(input).change(function(){
                              var key = $(this).val();
                              //console.log("change to " + key);
                              if ($(this).is(":checked")) {
                                  //console.log("it is checked");
                                  daSubscribedRoles[key] = 1;
                              }
                              else{
                                  //console.log("it is not checked");
                                  if (key in daSubscribedRoles){
                                      delete daSubscribedRoles[key];
                                  }
                              }
                              daUpdateMonitor();
                          });
                      }
                      else{
                          var input = $("#role" + skey).first();
                          if (key in newSubscribedRoles){
                              $(input).prop('checked', true);
                          }
                          else{
                              $(input).prop('checked', false);
                          }
                      }
                  }
                  daSubscribedRoles = newSubscribedRoles;
                  newDaSessions = Object();
                  for (var key in data.sessions){
                      if (data.sessions.hasOwnProperty(key)){
                          var user_id = key.replace(/^.*:userid:/, '');
                          if (true || user_id != daUserid){
                              var obj = data.sessions[key];
                              newDaSessions[key] = obj;
                              daDrawSession(key, obj);
                          }
                      }
                  }
                  var toDelete = Array();
                  var numSessions = 0;
                  for (var key in daSessions){
                      if (daSessions.hasOwnProperty(key)){
                          numSessions++;
                          if (!(key in newDaSessions)){
                              toDelete.push(key);
                          }
                      }
                  }
                  for (var i = 0; i < toDelete.length; ++i){
                      var key = toDelete[i];
                      daUndrawSession(key);
                  }
                  if ($("#monitorsessions").find("li").length > 0){
                      $("#emptylist").addClass("dainvisible");
                  }
                  else{
                      $("#emptylist").removeClass("dainvisible");
                  }
              });
          }
          if (daAvailableForChat){
              $("#daNotAvailable").addClass("dainvisible");
              daCheckNotifications();
          }
          else{
              $("#daAvailable").addClass("dainvisible");
          }
          $("#daAvailable").click(function(event){
              $("#daAvailable").addClass("dainvisible");
              $("#daNotAvailable").removeClass("dainvisible");
              daAvailableForChat = false;
              //console.log("daAvailableForChat: " + daAvailableForChat);
              daUpdateMonitor();
              daPlaySound('signinout');
          });
          $("#daNotAvailable").click(function(event){
              daCheckNotifications();
              $("#daNotAvailable").addClass("dainvisible");
              $("#daAvailable").removeClass("dainvisible");
              daAvailableForChat = true;
              //console.log("daAvailableForChat: " + daAvailableForChat);
              daUpdateMonitor();
              daPlaySound('signinout');
          });
          $( window ).bind('unload', function() {
            if (typeof daSocket !== 'undefined'){
              daSocket.emit('terminate');
            }
          });
          if (daUsePhone){
            $("#daPhoneInfo").removeClass("dainvisible");
            $("#daPhoneNumber").val(daPhoneNumber);
            $("#daPhoneNumber").change(daCheckPhone);
            $("#daPhoneNumber").bind('keypress keydown keyup', function(e){
              var theCode = e.which || e.keyCode;
              if(theCode == 13) { $(this).blur(); e.preventDefault(); }
            });
          }
          $(window).on('scroll', daOnScrollResize);
          $(window).on('resize', daOnScrollResize);
          $(".da-chat-notifier").click(function(e){
              //var key = $(this).data('key');
              var direction = 0;
              if ($(this).attr('id') == "chat-message-above"){
                  direction = -1;
              }
              else{
                  direction = 1;
              }
              var target = -1;
              var targetElement = null;
              for (var key in daUpdatedSessions){
                  if (daUpdatedSessions.hasOwnProperty(key)){
                      var top = $(key).offset().top;
                      if (direction == -1){
                          if (target == -1 || top < target){
                              target = top;
                              targetElement = key;
                          }
                      }
                      else{
                          if (target == -1 || top > target){
                              target = top;
                              targetElement = key;
                          }
                      }
                  }
              }
              if (target >= 0){
                  $("html, body").animate({scrollTop: target - 60}, 500, function(){
                      $(targetElement).find("input").first().focus();
                  });
              }
              e.preventDefault();
              return false;
          });
      });
    </script>"""
    response = make_response(render_template('pages/monitor.html', version_warning=None, bodyclass='daadminbody', extra_js=Markup(script), tab_title=word('Monitor'), page_title=word('Monitor')), 200)
    response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
    return response

@app.route('/updatingpackages', methods=['GET', 'POST'])
@login_required
@roles_required(['admin', 'developer'])
def update_package_wait():
    setup_translation()
    next_url = app.user_manager.make_safe_url_function(request.args.get('next', url_for('update_package')))
    my_csrf = generate_csrf()
    script = """
    <script>
      var daCheckinInterval = null;
      var resultsAreIn = false;
      var pollDelay = 0;
      var pollFail = 0;
      var pollPending = false;
      function daRestartCallback(data){
        //console.log("Restart result: " + data.success);
      }
      function daRestart(){
        $.ajax({
          type: 'POST',
          url: """ + json.dumps(url_for('restart_ajax')) + """,
          data: 'csrf_token=""" + my_csrf + """&action=restart',
          success: daRestartCallback,
          dataType: 'json'
        });
        return true;
      }
      function daBadCallback(data){
        pollPending = false;
        pollFail += 1;
      }
      function daUpdateCallback(data){
        pollPending = false;
        if (data.success){
          if (data.status == 'finished'){
            resultsAreIn = true;
            if (data.ok){
              $("#notification").html(""" + json.dumps(word("The package update did not report an error.  The logs are below.")) + """);
              $("#notification").removeClass("alert-info");
              $("#notification").removeClass("alert-danger");
              $("#notification").addClass("alert-success");
            }
            else{
              $("#notification").html(""" + json.dumps(word("The package update reported an error.  The logs are below.")) + """);
              $("#notification").removeClass("alert-info");
              $("#notification").removeClass("alert-success");
              $("#notification").addClass("alert-danger");
            }
            $("#resultsContainer").show();
            $("#resultsArea").html(data.summary);
            if (daCheckinInterval != null){
              clearInterval(daCheckinInterval);
            }
            //daRestart();
          }
          else if (data.status == 'failed' && !resultsAreIn){
            resultsAreIn = true;
            $("#notification").html(""" + json.dumps(word("There was an error updating the packages.")) + """);
            $("#notification").removeClass("alert-info");
            $("#notification").removeClass("alert-success");
            $("#notification").addClass("alert-danger");
            $("#resultsContainer").show();
            if (data.error_message){
              $("#resultsArea").html(data.error_message);
            }
            else if (data.summary){
              $("#resultsArea").html(data.summary);
            }
            if (daCheckinInterval != null){
              clearInterval(daCheckinInterval);
            }
          }
        }
        else if (!resultsAreIn){
          $("#notification").html(""" + json.dumps(word("There was an error.")) + """);
          $("#notification").removeClass("alert-info");
          $("#notification").removeClass("alert-success");
          $("#notification").addClass("alert-danger");
          if (daCheckinInterval != null){
            clearInterval(daCheckinInterval);
          }
        }
      }
      function daUpdate(){
        if (pollDelay > 25 || pollFail > 8){
          $("#notification").html(""" + json.dumps(word("Server did not respond to request for update.")) + """);
          $("#notification").removeClass("alert-info");
          $("#notification").removeClass("alert-success");
          $("#notification").addClass("alert-danger");
          if (daCheckinInterval != null){
            clearInterval(daCheckinInterval);
          }
          return;
        }
        if (pollPending){
          pollDelay += 1;
          return;
        }
        if (resultsAreIn){
          return;
        }
        pollDelay = 0;
        pollPending = true;
        $.ajax({
          type: 'POST',
          url: """ + json.dumps(url_for('update_package_ajax')) + """,
          data: 'csrf_token=""" + my_csrf + """',
          success: daUpdateCallback,
          error: daBadCallback,
          timeout: 10000,
          dataType: 'json'
        });
        return true;
      }
      $( document ).ready(function() {
        //console.log("page loaded");
        daCheckinInterval = setInterval(daUpdate, 6000);
      });
    </script>"""
    response = make_response(render_template('pages/update_package_wait.html', version_warning=None, bodyclass='daadminbody', extra_js=Markup(script), tab_title=word('Updating'), page_title=word('Updating'), next_page=next_url), 200)
    response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
    return response

@app.route('/update_package_ajax', methods=['POST'])
@login_required
@roles_required(['admin', 'developer'])
def update_package_ajax():
    if 'taskwait' not in session or 'serverstarttime' not in session:
        return jsonify(success=False)
    setup_translation()
    result = docassemble.webapp.worker.workerapp.AsyncResult(id=session['taskwait'])
    if result.ready():
        #if 'taskwait' in session:
        #    del session['taskwait']
        the_result = result.get()
        if isinstance(the_result, ReturnValue):
            if the_result.ok:
                #logmessage("update_package_ajax: success")
                if (hasattr(the_result, 'restart') and not the_result.restart) or START_TIME > session['serverstarttime']:
                    return jsonify(success=True, status='finished', ok=the_result.ok, summary=summarize_results(the_result.results, the_result.logmessages))
                else:
                    return jsonify(success=True, status='waiting')
            if hasattr(the_result, 'error_message'):
                logmessage("update_package_ajax: failed return value is " + str(the_result.error_message))
                return jsonify(success=True, status='failed', error_message=str(the_result.error_message))
            if hasattr(the_result, 'results') and hasattr(the_result, 'logmessages'):
                return jsonify(success=True, status='failed', summary=summarize_results(the_result.results, the_result.logmessages))
            return jsonify(success=True, status='failed', error_message=str("No error message.  Result is " + str(the_result)))
        else:
            logmessage("update_package_ajax: failed return value is a " + str(type(the_result)))
            logmessage("update_package_ajax: failed return value is " + str(the_result))
            return jsonify(success=True, status='failed', error_message=str(the_result))
    return jsonify(success=True, status='waiting')

def get_package_name_from_zip(zippath):
    with zipfile.ZipFile(zippath, mode='r') as zf:
        min_level = 999999
        setup_py = None
        for zinfo in zf.infolist():
            parts = splitall(zinfo.filename)
            if parts[-1] == 'setup.py':
                if len(parts) < min_level:
                    setup_py = zinfo
                    min_level = len(parts)
        if setup_py is None:
            raise Exception("Not a Python package zip file")
        with zf.open(setup_py) as f:
            the_file = TextIOWrapper(f, encoding='utf8')
            contents = the_file.read()
    extracted = {}
    for line in contents.splitlines():
        m = re.search(r"^NAME *= *\(?'(.*)'", line)
        if m:
            extracted['name'] = m.group(1)
        m = re.search(r'^NAME *= *\(?"(.*)"', line)
        if m:
            extracted['name'] = m.group(1)
        m = re.search(r'^NAME *= *\[(.*)\]', line)
        if m:
            extracted['name'] = m.group(1)
    if 'name' in extracted:
        return extracted['name']
    contents = re.sub(r'.*setup\(', '', contents, flags=re.DOTALL)
    extracted = {}
    for line in contents.splitlines():
        m = re.search(r"^ *([a-z_]+) *= *\(?'(.*)'", line)
        if m:
            extracted[m.group(1)] = m.group(2)
        m = re.search(r'^ *([a-z_]+) *= *\(?"(.*)"', line)
        if m:
            extracted[m.group(1)] = m.group(2)
        m = re.search(r'^ *([a-z_]+) *= *\[(.*)\]', line)
        if m:
            extracted[m.group(1)] = m.group(2)
    if 'name' not in extracted:
        raise Exception("Could not find name of Python package")
    return extracted['name']

@app.route('/updatepackage', methods=['GET', 'POST'])
@login_required
@roles_required(['admin', 'developer'])
def update_package():
    setup_translation()
    if not app.config['ALLOW_UPDATES']:
        return ('File not found', 404)
    if 'taskwait' in session:
        del session['taskwait']
    if 'serverstarttime' in session:
        del session['serverstarttime']
    #pip.utils.logging._log_state = threading.local()
    #pip.utils.logging._log_state.indentation = 0
    if request.method == 'GET' and app.config['USE_GITHUB'] and r.get('da:using_github:userid:' + str(current_user.id)) is not None:
        storage = RedisCredStorage(app='github')
        credentials = storage.get()
        if not credentials or credentials.invalid:
            state_string = random_string(16)
            session['github_next'] = json.dumps(dict(state=state_string, path='playground_packages', arguments=request.args))
            flow = get_github_flow()
            uri = flow.step1_get_authorize_url(state=state_string)
            return redirect(uri)
    form = UpdatePackageForm(request.form)
    form.gitbranch.choices = [('', "Not applicable")]
    if form.gitbranch.data:
        form.gitbranch.choices.append((form.gitbranch.data, form.gitbranch.data))
    action = request.args.get('action', None)
    target = request.args.get('package', None)
    limitation = request.args.get('limitation', '')
    branch = None
    if action is not None and target is not None:
        package_list, package_auth = get_package_info()
        the_package = None
        for package in package_list:
            if package.package.name == target:
                the_package = package
                break
        if the_package is not None:
            if action == 'uninstall' and the_package.can_uninstall:
                uninstall_package(target)
            elif action == 'update' and the_package.can_update:
                existing_package = db.session.execute(select(Package).filter_by(name=target, active=True).order_by(Package.id.desc())).scalar()
                if existing_package is not None:
                    if limitation and existing_package.limitation != limitation:
                        existing_package.limitation = limitation
                        db.session.commit()
                    if existing_package.type == 'git' and existing_package.giturl is not None:
                        if existing_package.gitbranch:
                            install_git_package(target, existing_package.giturl, existing_package.gitbranch)
                        else:
                            install_git_package(target, existing_package.giturl, get_master_branch(existing_package.giturl))
                    elif existing_package.type == 'pip':
                        if existing_package.name == 'docassemble.webapp' and existing_package.limitation and not limitation:
                            existing_package.limitation = None
                            db.session.commit()
                        install_pip_package(existing_package.name, existing_package.limitation)
        result = docassemble.webapp.worker.update_packages.apply_async(link=docassemble.webapp.worker.reset_server.s(run_create=should_run_create(target)))
        session['taskwait'] = result.id
        session['serverstarttime'] = START_TIME
        return redirect(url_for('update_package_wait'))
    if request.method == 'POST' and form.validate_on_submit():
        #use_pip_cache = form.use_cache.data
        #pipe = r.pipeline()
        #pipe.set('da:updatepackage:use_pip_cache', 1 if use_pip_cache else 0)
        #pipe.expire('da:updatepackage:use_pip_cache', 120)
        #pipe.execute()
        if 'zipfile' in request.files and request.files['zipfile'].filename:
            try:
                the_file = request.files['zipfile']
                filename = secure_filename(the_file.filename)
                file_number = get_new_file_number(None, filename)
                saved_file = SavedFile(file_number, extension='zip', fix=True, should_not_exist=True)
                file_set_attributes(file_number, private=False, persistent=True)
                zippath = saved_file.path
                the_file.save(zippath)
                saved_file.save()
                saved_file.finalize()
                pkgname = get_package_name_from_zip(zippath)
                if user_can_edit_package(pkgname=pkgname):
                    install_zip_package(pkgname, file_number)
                    result = docassemble.webapp.worker.update_packages.apply_async(link=docassemble.webapp.worker.reset_server.s(run_create=should_run_create(pkgname)))
                    session['taskwait'] = result.id
                    session['serverstarttime'] = START_TIME
                    return redirect(url_for('update_package_wait'))
                else:
                    flash(word("You do not have permission to install this package."), 'error')
            except Exception as errMess:
                flash("Error of type " + str(type(errMess)) + " processing upload: " + str(errMess), "error")
        else:
            if form.giturl.data:
                giturl = form.giturl.data.strip().rstrip('/')
                branch = form.gitbranch.data.strip()
                if not branch:
                    branch = get_master_branch(giturl)
                packagename = re.sub(r'/*$', '', giturl)
                packagename = re.sub(r'^git+', '', packagename)
                packagename = re.sub(r'#.*', '', packagename)
                packagename = re.sub(r'\.git$', '', packagename)
                packagename = re.sub(r'.*/', '', packagename)
                packagename = re.sub(r'^docassemble-', 'docassemble.', packagename)
                if user_can_edit_package(giturl=giturl) and user_can_edit_package(pkgname=packagename):
                    install_git_package(packagename, giturl, branch)
                    result = docassemble.webapp.worker.update_packages.apply_async(link=docassemble.webapp.worker.reset_server.s(run_create=should_run_create(packagename)))
                    session['taskwait'] = result.id
                    session['serverstarttime'] = START_TIME
                    return redirect(url_for('update_package_wait'))
                else:
                    flash(word("You do not have permission to install this package."), 'error')
            elif form.pippackage.data:
                m = re.match(r'([^>=<]+)([>=<]+.+)', form.pippackage.data)
                if m:
                    packagename = m.group(1)
                    limitation = m.group(2)
                else:
                    packagename = form.pippackage.data
                    limitation = None
                packagename = re.sub(r'[^A-Za-z0-9\_\-\.]', '', packagename)
                if user_can_edit_package(pkgname=packagename):
                    install_pip_package(packagename, limitation)
                    result = docassemble.webapp.worker.update_packages.apply_async(link=docassemble.webapp.worker.reset_server.s(run_create=should_run_create(packagename)))
                    session['taskwait'] = result.id
                    session['serverstarttime'] = START_TIME
                    return redirect(url_for('update_package_wait'))
                else:
                    flash(word("You do not have permission to install this package."), 'error')
            else:
                flash(word('You need to supply a Git URL, upload a file, or supply the name of a package on PyPI.'), 'error')
    package_list, package_auth = get_package_info(exclude_core=True)
    form.pippackage.data = None
    form.giturl.data = None
    extra_js = """
    <script>
      var default_branch = """ + json.dumps(branch if branch else 'null') + """;
      function get_branches(){
        var elem = $("#gitbranch");
        elem.empty();
        var opt = $("<option><\/option>");
        opt.attr("value", "").text("Not applicable");
        elem.append(opt);
        var github_url = $("#giturl").val();
        if (!github_url){
          return;
        }
        $.get(""" + json.dumps(url_for('get_git_branches')) + """, { url: github_url }, "json")
        .done(function(data){
          //console.log(data);
          if (data.success){
            var n = data.result.length;
            if (n > 0){
              var default_to_use = default_branch;
              var to_try = [default_branch, """ + json.dumps(GITHUB_BRANCH) + """, 'master', 'main'];
            outer:
              for (var j = 0; j < 4; j++){
                for (var i = 0; i < n; i++){
                  if (data.result[i].name == to_try[j]){
                    default_to_use = to_try[j];
                    break outer;
                  }
                }
              }
              elem.empty();
              for (var i = 0; i < n; i++){
                opt = $("<option><\/option>");
                opt.attr("value", data.result[i].name).text(data.result[i].name);
                if (data.result[i].name == default_to_use){
                  opt.prop('selected', true);
                }
                $(elem).append(opt);
              }
            }
          }
        });
      }
      $( document ).ready(function() {
        get_branches();
        $("#giturl").on('change', get_branches);
      });
      $('#zipfile').on('change', function(){
        var fileName = $(this).val();
        fileName = fileName.replace(/.*\\\\/, '');
        fileName = fileName.replace(/.*\\//, '');
        $(this).next('.custom-file-label').html(fileName);
      });
    </script>"""
    python_version = daconfig.get('python version', word('Unknown'))
    version = word("Current") + ': <span class="badge bg-primary">' + str(python_version) + '</span>'
    dw_status = pypi_status('docassemble.webapp')
    if daconfig.get('stable version', False):
        if not dw_status['error'] and 'info' in dw_status and 'releases' in dw_status['info'] and isinstance(dw_status['info']['releases'], dict):
            stable_version = LooseVersion('1.1')
            latest_version = None
            for version_number, version_info in dw_status['info']['releases'].items():
                version_number_loose = LooseVersion(version_number)
                if version_number_loose >= stable_version:
                    continue
                if latest_version is None or version_number_loose > LooseVersion(latest_version):
                    latest_version = version_number
            if latest_version != str(python_version):
                version += ' ' + word("Available") + ': <span class="badge bg-success">' + latest_version + '</span>'
    else:
        if not dw_status['error'] and 'info' in dw_status and 'info' in dw_status['info'] and 'version' in dw_status['info']['info'] and dw_status['info']['info']['version'] != str(python_version):
            version += ' ' + word("Available") + ': <span class="badge bg-success">' + dw_status['info']['info']['version'] + '</span>'
    allowed_to_upgrade = current_user.has_role('admin') or user_can_edit_package(pkgname='docassemble.webapp')
    if daconfig.get('stable version', False):
        limitation = '<1.1'
    else:
        limitation = ''
    if daconfig.get('stable version', False):
        limitation = '<1.1.0'
    else:
        limitation = ''
    allowed_to_upgrade = current_user.has_role('admin') or user_can_edit_package(pkgname='docassemble.webapp')
    response = make_response(render_template('pages/update_package.html', version_warning=version_warning, bodyclass='daadminbody', form=form, package_list=sorted(package_list, key=lambda y: (0 if y.package.name.startswith('docassemble') else 1, y.package.name.lower())), tab_title=word('Package Management'), page_title=word('Package Management'), extra_js=Markup(extra_js), version=Markup(version), allowed_to_upgrade=allowed_to_upgrade, limitation=limitation), 200)
    response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
    return response

def get_master_branch(giturl):
    try:
        return get_repo_info(giturl).get('default_branch', GITHUB_BRANCH)
    except:
        return GITHUB_BRANCH

# @app.route('/testws', methods=['GET', 'POST'])
# def test_websocket():
#     script = '<script type="text/javascript" src="' + url_for('static', filename='app/socket.io.min.js') + '"></script>' + """<script type="text/javascript" charset="utf-8">
#     var daSocket;
#     $(document).ready(function(){
#         if (location.protocol === 'http:' || document.location.protocol === 'http:'){
#             daSocket = io.connect("http://" + document.domain + "/wsinterview", {path: '/ws/socket.io'});
#         }
#         if (location.protocol === 'https:' || document.location.protocol === 'https:'){
#             daSocket = io.connect("https://" + document.domain + "/wsinterview" + location.port, {path: '/ws/socket.io'});
#         }
#         if (typeof daSocket !== 'undefined') {
#             daSocket.on('connect', function() {
#                 //console.log("Connected!");
#                 daSocket.emit('chat_log', {data: 1});
#             });
#             daSocket.on('mymessage', function(arg) {
#                 //console.log("Received " + arg.data);
#                 $("#daPushResult").html(arg.data);
#             });
#             daSocket.on('chatmessage', function(arg) {
#                 console.log("Received chat message " + arg.data);
#                 var newDiv = document.createElement('div');
#                 $(newDiv).html(arg.data.message);
#                 $("#daCorrespondence").append(newDiv);
#             });
#         }
#         $("#daSend").click(daSender);
#     });
# </script>"""
#     return render_template('pages/socketserver.html', extra_js=Markup(script)), 200

@app.route('/createplaygroundpackage', methods=['GET', 'POST'])
@login_required
@roles_required(['admin', 'developer'])
def create_playground_package():
    setup_translation()
    if not app.config['ENABLE_PLAYGROUND']:
        return ('File not found', 404)
    fix_package_folder()
    current_project = get_current_project()
    form = CreatePlaygroundPackageForm(request.form)
    current_package = request.args.get('package', None)
    if current_package is not None:
        current_package = werkzeug.utils.secure_filename(current_package)
    do_pypi = request.args.get('pypi', False)
    do_github = request.args.get('github', False)
    do_install = request.args.get('install', False)
    branch = request.args.get('branch', None)
    if branch is not None:
        branch = branch.strip()
    if branch in ('', 'None'):
        branch = None
    new_branch = request.args.get('new_branch', None)
    if new_branch is not None and new_branch not in ('', 'None'):
        branch = new_branch
        branch_is_new = True
    else:
        branch_is_new = False
    force_branch_creation = False
    sanitize_arguments(do_pypi, do_github, do_install, branch, new_branch)
    if app.config['USE_GITHUB']:
        github_auth = r.get('da:using_github:userid:' + str(current_user.id))
    else:
        github_auth = None
    area = {}
    area['playgroundpackages'] = SavedFile(current_user.id, fix=True, section='playgroundpackages')
    if os.path.isfile(os.path.join(directory_for(area['playgroundpackages'], current_project), 'docassemble.' + current_package)):
        filename = os.path.join(directory_for(area['playgroundpackages'], current_project), 'docassemble.' + current_package)
        info = {}
        with open(filename, 'r', encoding='utf-8') as fp:
            content = fp.read()
            info = yaml.load(content, Loader=yaml.FullLoader)
    else:
        info = {}
    if do_github:
        if not app.config['USE_GITHUB']:
            return ('File not found', 404)
        if current_package is None:
            logmessage('create_playground_package: package not specified')
            return ('File not found', 404)
        if not github_auth:
            logmessage('create_playground_package: github button called when github auth not enabled.')
            return ('File not found', 404)
        github_auth = github_auth.decode()
        if github_auth == '1':
            github_auth_info = dict(shared=True, orgs=True)
        else:
            github_auth_info = json.loads(github_auth)
        github_package_name = 'docassemble-' + re.sub(r'^docassemble-', r'', current_package)
        #github_package_name = re.sub(r'[^A-Za-z\_\-]', '', github_package_name)
        if github_package_name in ('docassemble-base', 'docassemble-webapp', 'docassemble-demo'):
            return ('File not found', 404)
        commit_message = request.args.get('commit_message', 'a commit')
        storage = RedisCredStorage(app='github')
        credentials = storage.get()
        if not credentials or credentials.invalid:
            state_string = random_string(16)
            session['github_next'] = json.dumps(dict(state=state_string, path='create_playground_package', arguments=request.args))
            flow = get_github_flow()
            uri = flow.step1_get_authorize_url(state=state_string)
            return redirect(uri)
        http = credentials.authorize(httplib2.Http())
        resp, content = http.request("https://api.github.com/user", "GET")
        if int(resp['status']) == 200:
            user_info = json.loads(content.decode())
            github_user_name = user_info.get('login', None)
            github_email = user_info.get('email', None)
        else:
            raise DAError("create_playground_package: could not get information about GitHub User")
        if github_email is None:
            resp, content = http.request("https://api.github.com/user/emails", "GET")
            if int(resp['status']) == 200:
                email_info = json.loads(content.decode())
                for item in email_info:
                    if item.get('email', None) and item.get('visibility', None) != 'private':
                        github_email = item['email']
        if github_user_name is None or github_email is None:
            raise DAError("create_playground_package: login and/or email not present in user info from GitHub")
        github_url_from_file = info.get('github_url', None)
        found = False
        found_strong = False
        commit_repository = None
        resp, content = http.request("https://api.github.com/repos/" + str(github_user_name) + "/" + github_package_name, "GET")
        if int(resp['status']) == 200:
            repo_info = json.loads(content.decode('utf-8', 'ignore'))
            commit_repository = repo_info
            found = True
            if github_url_from_file is None or github_url_from_file in [repo_info['html_url'], repo_info['ssh_url']]:
                found_strong = True
        if found_strong is False and github_auth_info['shared']:
            repositories = get_user_repositories(http)
            for repo_info in repositories:
                if repo_info['name'] != github_package_name or (commit_repository is not None and commit_repository.get('html_url', None) is not None and commit_repository['html_url'] == repo_info['html_url']) or (commit_repository is not None and commit_repository.get('ssh_url', None) is not None and commit_repository['ssh_url'] == repo_info['ssh_url']):
                    continue
                if found and github_url_from_file is not None and github_url_from_file not in [repo_info['html_url'], repo_info['ssh_url']]:
                    break
                commit_repository = repo_info
                found = True
                if github_url_from_file is None or github_url_from_file in [repo_info['html_url'], repo_info['ssh_url']]:
                    found_strong = True
                break
        if found_strong is False and github_auth_info['orgs']:
            orgs_info = get_orgs_info(http)
            for org_info in orgs_info:
                resp, content = http.request("https://api.github.com/repos/" + str(org_info['login']) + "/" + github_package_name, "GET")
                if int(resp['status']) == 200:
                    repo_info = json.loads(content.decode('utf-8', 'ignore'))
                    if found and github_url_from_file is not None and github_url_from_file not in [repo_info['html_url'], repo_info['ssh_url']]:
                        break
                    commit_repository = repo_info
                    break
    file_list = {}
    the_directory = directory_for(area['playgroundpackages'], current_project)
    file_list['playgroundpackages'] = sorted([re.sub(r'^docassemble.', r'', f) for f in os.listdir(the_directory) if os.path.isfile(os.path.join(the_directory, f)) and re.search(r'^[A-Za-z0-9]', f)])
    the_choices = []
    for file_option in file_list['playgroundpackages']:
        the_choices.append((file_option, file_option))
    form.name.choices = the_choices
    if request.method == 'POST':
        if form.validate():
            current_package = form.name.data
            #flash("form validated", 'success')
        else:
            the_error = ''
            for error in form.name.errors:
                the_error += str(error)
            flash("form did not validate with " + str(form.name.data) + " " + str(the_error) + " among " + str(form.name.choices), 'error')
    if current_package is not None:
        pkgname = re.sub(r'^docassemble-', r'', current_package)
        #if not user_can_edit_package(pkgname='docassemble.' + pkgname):
        #    flash(word('That package name is already in use by someone else.  Please change the name.'), 'error')
        #    current_package = None
    if current_package is not None and current_package not in file_list['playgroundpackages']:
        flash(word('Sorry, that package name does not exist in the playground'), 'error')
        current_package = None
    if current_package is not None:
        section_sec = {'playgroundtemplate': 'template', 'playgroundstatic': 'static', 'playgroundsources': 'sources', 'playgroundmodules': 'modules'}
        for sec in ('playground', 'playgroundtemplate', 'playgroundstatic', 'playgroundsources', 'playgroundmodules'):
            area[sec] = SavedFile(current_user.id, fix=True, section=sec)
            the_directory = directory_for(area[sec], current_project)
            file_list[sec] = sorted([f for f in os.listdir(the_directory) if os.path.isfile(os.path.join(the_directory, f)) and re.search(r'^[A-Za-z0-9]', f)])
        if os.path.isfile(os.path.join(directory_for(area['playgroundpackages'], current_project), 'docassemble.' + current_package)):
            filename = os.path.join(directory_for(area['playgroundpackages'], current_project), 'docassemble.' + current_package)
            info = {}
            with open(filename, 'r', encoding='utf-8') as fp:
                content = fp.read()
                info = yaml.load(content, Loader=yaml.FullLoader)
            for field in ('dependencies', 'interview_files', 'template_files', 'module_files', 'static_files', 'sources_files'):
                if field not in info:
                    info[field] = []
            info['dependencies'] = [x for x in [z for z in map(lambda y: re.sub(r'[\>\<\=].*', '', y), info['dependencies'])] if x not in ('docassemble', 'docassemble.base', 'docassemble.webapp')]
            info['modtime'] = os.path.getmtime(filename)
            author_info = {}
            author_info['author name and email'] = name_of_user(current_user, include_email=True)
            author_info['author name'] = name_of_user(current_user)
            author_info['author email'] = current_user.email
            author_info['first name'] = current_user.first_name
            author_info['last name'] = current_user.last_name
            author_info['id'] = current_user.id
            if do_pypi:
                if current_user.pypi_username is None or current_user.pypi_password is None or current_user.pypi_username == '' or current_user.pypi_password == '':
                    flash("Could not publish to PyPI because username and password were not defined")
                    return redirect(url_for('playground_packages', project=current_project, file=current_package))
                if current_user.timezone:
                    the_timezone = current_user.timezone
                else:
                    the_timezone = get_default_timezone()
                fix_ml_files(author_info['id'], current_project)
                had_error, logmessages = docassemble.webapp.files.publish_package(pkgname, info, author_info, current_project=current_project)
                flash(logmessages, 'danger' if had_error else 'info')
                if not do_install:
                    time.sleep(3.0)
                    return redirect(url_for('playground_packages', project=current_project, file=current_package))
            if do_github:
                if commit_repository is not None:
                    resp, content = http.request("https://api.github.com/repos/" + commit_repository['full_name'] + "/commits?per_page=1" , "GET")
                    if int(resp['status']) == 200:
                        commit_list = json.loads(content.decode('utf-8', 'ignore'))
                        if len(commit_list) == 0:
                            first_time = True
                            is_empty = True
                        else:
                            first_time = False
                            is_empty = False
                    else:
                        first_time = True
                        is_empty = True
                else:
                    first_time = True
                    is_empty = False
                    headers = {'Content-Type': 'application/json'}
                    the_license = 'mit' if re.search(r'MIT License', info.get('license', '')) else None
                    body = json.dumps(dict(name=github_package_name, description=info.get('description', None), homepage=info.get('url', None), license_template=the_license))
                    resp, content = http.request("https://api.github.com/user/repos", "POST", headers=headers, body=body)
                    if int(resp['status']) != 201:
                        raise DAError("create_playground_package: unable to create GitHub repository: status " + str(resp['status']) + " " + str(content))
                    resp, content = http.request("https://api.github.com/repos/" + str(github_user_name) + "/" + github_package_name, "GET")
                    if int(resp['status']) == 200:
                        commit_repository = json.loads(content.decode('utf-8', 'ignore'))
                    else:
                        raise DAError("create_playground_package: GitHub repository could not be found after creating it.")
                if first_time:
                    logmessage("Not checking for stored commit code because no target repository exists")
                    pulled_already = False
                else:
                    current_commit_file = os.path.join(directory_for(area['playgroundpackages'], current_project), '.' + github_package_name)
                    if os.path.isfile(current_commit_file):
                        with open(current_commit_file, 'r', encoding='utf-8') as fp:
                            commit_code = fp.read()
                        commit_code = commit_code.strip()
                        resp, content = http.request("https://api.github.com/repos/" + commit_repository['full_name'] + "/commits/" + commit_code , "GET")
                        if int(resp['status']) == 200:
                            logmessage("Stored commit code is valid")
                            pulled_already = True
                        else:
                            logmessage("Stored commit code is invalid")
                            pulled_already = False
                    else:
                        logmessage("Commit file not found")
                        pulled_already = False
                directory = tempfile.mkdtemp()
                (private_key_file, public_key_file) = get_ssh_keys(github_email)
                os.chmod(private_key_file, stat.S_IRUSR | stat.S_IWUSR)
                os.chmod(public_key_file, stat.S_IRUSR | stat.S_IWUSR)
                ssh_script = tempfile.NamedTemporaryFile(mode='w', prefix="datemp", suffix='.sh', delete=False, encoding='utf-8')
                ssh_script.write('# /bin/bash\n\nssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o GlobalKnownHostsFile=/dev/null -i "' + str(private_key_file) + '" $1 $2 $3 $4 $5 $6')
                ssh_script.close()
                os.chmod(ssh_script.name, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR )
                #git_prefix = "GIT_SSH_COMMAND='ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o GlobalKnownHostsFile=/dev/null -i \"" + str(private_key_file) + "\"' "
                git_prefix = "GIT_SSH=" + ssh_script.name + " "
                ssh_url = commit_repository.get('ssh_url', None)
                github_url = commit_repository.get('html_url', None)
                commit_branch = commit_repository.get('default_branch', GITHUB_BRANCH)
                if ssh_url is None:
                    raise DAError("create_playground_package: could not obtain ssh_url for package")
                output = ''
                if branch:
                    branch_option = '-b ' + str(branch) + ' '
                else:
                    branch_option = '-b ' + commit_branch + ' '
                tempbranch = 'playground' + random_string(5)
                packagedir = os.path.join(directory, 'docassemble-' + str(pkgname))
                the_user_name = str(current_user.first_name) + " " + str(current_user.last_name)
                if the_user_name == ' ':
                    the_user_name = 'Anonymous User'
                if is_empty:
                    os.makedirs(packagedir)
                    output += "Doing git init\n"
                    try:
                        output += subprocess.check_output(["git", "init"], cwd=packagedir, stderr=subprocess.STDOUT).decode()
                    except subprocess.CalledProcessError as err:
                        output += err.output
                        raise DAError("create_playground_package: error running git init.  " + output)
                    with open(os.path.join(packagedir, 'README.md'), 'w', encoding='utf-8') as the_file:
                        the_file.write("")
                    output += "Doing git config user.email " + json.dumps(github_email) + "\n"
                    try:
                        output += subprocess.check_output(["git", "config", "user.email", json.dumps(github_email)], cwd=packagedir, stderr=subprocess.STDOUT).decode()
                    except subprocess.CalledProcessError as err:
                        output += err.output.decode()
                        raise DAError("create_playground_package: error running git config user.email.  " + output)
                    output += "Doing git config user.name " + json.dumps(the_user_name) + "\n"
                    try:
                        output += subprocess.check_output(["git", "config", "user.name", json.dumps(the_user_name)], cwd=packagedir, stderr=subprocess.STDOUT).decode()
                    except subprocess.CalledProcessError as err:
                        output += err.output.decode()
                        raise DAError("create_playground_package: error running git config user.name.  " + output)
                    output += "Doing git add README.MD\n"
                    try:
                        output += subprocess.check_output(["git", "add", "README.md"], cwd=packagedir, stderr=subprocess.STDOUT).decode()
                    except subprocess.CalledProcessError as err:
                        output += err.output.decode()
                        raise DAError("create_playground_package: error running git add README.md.  " + output)
                    output += "Doing git commit -m \"first commit\"\n"
                    try:
                        output += subprocess.check_output(["git", "commit", "-m", "first commit"], cwd=packagedir, stderr=subprocess.STDOUT).decode()
                    except subprocess.CalledProcessError as err:
                        output += err.output.decode()
                        raise DAError("create_playground_package: error running git commit -m \"first commit\".  " + output)
                    output += "Doing git branch -M " + commit_branch + "\n"
                    try:
                        output += subprocess.check_output(["git", "branch", "-M", commit_branch], cwd=packagedir, stderr=subprocess.STDOUT).decode()
                    except subprocess.CalledProcessError as err:
                        output += err.output.decode()
                        raise DAError("create_playground_package: error running git branch -M " + commit_branch + ".  " + output)
                    output += "Doing git remote add origin " + ssh_url + "\n"
                    try:
                        output += subprocess.check_output(["git", "remote", "add", "origin", ssh_url], cwd=packagedir, stderr=subprocess.STDOUT).decode()
                    except subprocess.CalledProcessError as err:
                        output += err.output.decode()
                        raise DAError("create_playground_package: error running git remote add origin.  " + output)
                    output += "Doing " + git_prefix + "git push -u origin " + '"' + commit_branch + '"' + "\n"
                    try:
                        output += subprocess.check_output(git_prefix + "git push -u origin " + '"' + commit_branch + '"', cwd=packagedir, stderr=subprocess.STDOUT, shell=True).decode()
                    except subprocess.CalledProcessError as err:
                        output += err.output.decode()
                        raise DAError("create_playground_package: error running first git push.  " + output)
                else:
                    output += "Doing " + git_prefix + "git clone " + ssh_url + "\n"
                    try:
                        output += subprocess.check_output(git_prefix + "git clone " + ssh_url, cwd=directory, stderr=subprocess.STDOUT, shell=True).decode()
                    except subprocess.CalledProcessError as err:
                        output += err.output.decode()
                        raise DAError("create_playground_package: error running git clone.  " + output)
                if not os.path.isdir(packagedir):
                    raise DAError("create_playground_package: package directory did not exist")
                if pulled_already:
                    output += "Doing git checkout " + commit_code + "\n"
                    try:
                        output += subprocess.check_output(git_prefix + "git checkout " + '"' + commit_code + '"', cwd=packagedir, stderr=subprocess.STDOUT, shell=True).decode()
                    except subprocess.CalledProcessError as err:
                        output += err.output.decode()
                        #raise DAError("create_playground_package: error running git checkout.  " + output)
                if current_user.timezone:
                    the_timezone = current_user.timezone
                else:
                    the_timezone = get_default_timezone()
                fix_ml_files(author_info['id'], current_project)
                docassemble.webapp.files.make_package_dir(pkgname, info, author_info, directory=directory, current_project=current_project)
                if branch:
                    the_branch = branch
                else:
                    the_branch = commit_branch
                output += "Going to use " + the_branch + " as the branch.\n"
                if not is_empty:
                    output += "Doing git config user.email " + json.dumps(github_email) + "\n"
                    try:
                        output += subprocess.check_output(["git", "config", "user.email", json.dumps(github_email)], cwd=packagedir, stderr=subprocess.STDOUT).decode()
                    except subprocess.CalledProcessError as err:
                        output += err.output.decode()
                        raise DAError("create_playground_package: error running git config user.email.  " + output)
                    output += "Doing git config user.name " + json.dumps(the_user_name) + "\n"
                    try:
                        output += subprocess.check_output(["git", "config", "user.name", json.dumps(the_user_name)], cwd=packagedir, stderr=subprocess.STDOUT).decode()
                    except subprocess.CalledProcessError as err:
                        output += err.output.decode()
                        raise DAError("create_playground_package: error running git config user.email.  " + output)
                    output += "Trying git checkout " + the_branch + "\n"
                    try:
                        output += subprocess.check_output(["git", "checkout", the_branch], cwd=packagedir, stderr=subprocess.STDOUT).decode()
                    except subprocess.CalledProcessError as err:
                        output += the_branch + " is a new branch\n"
                        force_branch_creation = True
                        branch = the_branch
                output += "Doing git checkout -b " + tempbranch + "\n"
                try:
                    output += subprocess.check_output(git_prefix + "git checkout -b " + tempbranch, cwd=packagedir, stderr=subprocess.STDOUT, shell=True).decode()
                except subprocess.CalledProcessError as err:
                    output += err.output.decode()
                    raise DAError("create_playground_package: error running git checkout.  " + output)
                output += "Doing git add .\n"
                try:
                    output += subprocess.check_output(["git", "add", "."], cwd=packagedir, stderr=subprocess.STDOUT).decode()
                except subprocess.CalledProcessError as err:
                    output += err.output
                    raise DAError("create_playground_package: error running git add.  " + output)
                output += "Doing git status\n"
                try:
                    output += subprocess.check_output(["git", "status"], cwd=packagedir, stderr=subprocess.STDOUT).decode()
                except subprocess.CalledProcessError as err:
                    output += err.output.decode()
                    raise DAError("create_playground_package: error running git status.  " + output)
                output += "Doing git commit -m " + json.dumps(str(commit_message)) + "\n"
                try:
                    output += subprocess.check_output(["git", "commit", "-am", str(commit_message)], cwd=packagedir, stderr=subprocess.STDOUT).decode()
                except subprocess.CalledProcessError as err:
                    output += err.output.decode()
                    raise DAError("create_playground_package: error running git commit.  " + output)
                output += "Trying git checkout " + the_branch + "\n"
                try:
                    output += subprocess.check_output(git_prefix + "git checkout " + '"' + the_branch + '"', cwd=packagedir, stderr=subprocess.STDOUT, shell=True).decode()
                    branch_exists = True
                except subprocess.CalledProcessError as err:
                    branch_exists = False
                if not branch_exists:
                    output += "Doing git checkout -b " + the_branch + "\n"
                    try:
                        output += subprocess.check_output(git_prefix + "git checkout -b " + '"' + the_branch + '"', cwd=packagedir, stderr=subprocess.STDOUT, shell=True).decode()
                    except subprocess.CalledProcessError as err:
                        output += err.output.decode()
                        raise DAError("create_playground_package: error running git checkout -b " + the_branch + ".  " + output)
                else:
                    output += "Doing git merge --squash " + tempbranch + "\n"
                    try:
                        output += subprocess.check_output(git_prefix + "git merge --squash " + tempbranch, cwd=packagedir, stderr=subprocess.STDOUT, shell=True).decode()
                    except subprocess.CalledProcessError as err:
                        output += err.output.decode()
                        raise DAError("create_playground_package: error running git merge --squash " + tempbranch + ".  " + output)
                    output += "Doing git commit\n"
                    try:
                        output += subprocess.check_output(["git", "commit", "-am", str(commit_message)], cwd=packagedir, stderr=subprocess.STDOUT).decode()
                    except subprocess.CalledProcessError as err:
                        output += err.output.decode()
                        raise DAError("create_playground_package: error running git commit -am " + str(commit_message) + ".  " + output)
                if branch:
                    output += "Doing " + git_prefix + "git push --set-upstream origin " + str(branch) + "\n"
                    try:
                        output += subprocess.check_output(git_prefix + "git push --set-upstream origin " + '"' + str(branch) + '"', cwd=packagedir, stderr=subprocess.STDOUT, shell=True).decode()
                    except subprocess.CalledProcessError as err:
                        output += err.output.decode()
                        raise DAError("create_playground_package: error running git push.  " + output)
                else:
                    output += "Doing " + git_prefix + "git push\n"
                    try:
                        output += subprocess.check_output(git_prefix + "git push", cwd=packagedir, stderr=subprocess.STDOUT, shell=True).decode()
                    except subprocess.CalledProcessError as err:
                        output += err.output.decode()
                        raise DAError("create_playground_package: error running git push.  " + output)
                logmessage(output)
                flash(word("Pushed commit to GitHub.") + "<br>" + re.sub(r'[\n\r]+', '<br>', output), 'info')
                time.sleep(3.0)
                shutil.rmtree(directory)
                the_args = dict(project=current_project, pull='1', github_url=ssh_url, show_message='0')
                do_pypi_also = true_or_false(request.args.get('pypi_also', False))
                do_install_also = true_or_false(request.args.get('install_also', False))
                if do_pypi_also or do_install_also:
                    the_args['file'] = current_package
                    if do_pypi_also:
                        the_args['pypi_also'] = '1'
                    if do_install_also:
                        the_args['install_also'] = '1'
                if branch:
                    the_args['branch'] = branch
                return redirect(url_for('playground_packages', **the_args))
            nice_name = 'docassemble-' + str(pkgname) + '.zip'
            file_number = get_new_file_number(None, nice_name)
            file_set_attributes(file_number, private=False, persistent=True)
            saved_file = SavedFile(file_number, extension='zip', fix=True, should_not_exist=True)
            if current_user.timezone:
                the_timezone = current_user.timezone
            else:
                the_timezone = get_default_timezone()
            fix_ml_files(author_info['id'], current_project)
            zip_file = docassemble.webapp.files.make_package_zip(pkgname, info, author_info, the_timezone, current_project=current_project)
            saved_file.copy_from(zip_file.name)
            saved_file.finalize()
            if do_install:
                install_zip_package('docassemble.' + pkgname, file_number)
                result = docassemble.webapp.worker.update_packages.apply_async(link=docassemble.webapp.worker.reset_server.s(run_create=should_run_create('docassemble.' + pkgname)))
                session['taskwait'] = result.id
                session['serverstarttime'] = START_TIME
                return redirect(url_for('update_package_wait', next=url_for('playground_packages', project=current_project, file=current_package)))
                #return redirect(url_for('playground_packages', file=current_package))
            else:
                response = send_file(saved_file.path, mimetype='application/zip', as_attachment=True, attachment_filename=nice_name)
                response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
                return response
    response = make_response(render_template('pages/create_playground_package.html', current_project=current_project, version_warning=version_warning, bodyclass='daadminbody', form=form, current_package=current_package, package_names=file_list['playgroundpackages'], tab_title=word('Playground Packages'), page_title=word('Playground Packages')), 200)
    response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
    return response

@app.route('/createpackage', methods=['GET', 'POST'])
@login_required
@roles_required(['admin', 'developer'])
def create_package():
    setup_translation()
    if not app.config['ENABLE_PLAYGROUND']:
        return ('File not found', 404)
    form = CreatePackageForm(request.form)
    if request.method == 'POST' and form.validate():
        pkgname = re.sub(r'^docassemble-', r'', form.name.data)
        initpy = """\
try:
    __import__('pkg_resources').declare_namespace(__name__)
except ImportError:
    __path__ = __import__('pkgutil').extend_path(__path__, __name__)

"""
        licensetext = """\
The MIT License (MIT)

"""
        licensetext += 'Copyright (c) ' + str(datetime.datetime.now().year) + ' ' + str(name_of_user(current_user)) + """

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
        readme = '# docassemble.' + str(pkgname) + "\n\nA docassemble extension.\n\n## Author\n\n" + name_of_user(current_user, include_email=True) + "\n"
        manifestin = """\
include README.md
"""
        setupcfg = """\
[metadata]
description-file = README
"""
        setuppy = """\
import os
import sys
from setuptools import setup, find_packages
from fnmatch import fnmatchcase
from distutils2.util import convert_path

standard_exclude = ('*.pyc', '*~', '.*', '*.bak', '*.swp*')
standard_exclude_directories = ('.*', 'CVS', '_darcs', os.path.join('.', 'build'), os.path.join('.', 'dist'), 'EGG-INFO', '*.egg-info')
def find_package_data(where='.', package='', exclude=standard_exclude, exclude_directories=standard_exclude_directories):
    out = {}
    stack = [(convert_path(where), '', package)]
    while stack:
        where, prefix, package = stack.pop(0)
        for name in os.listdir(where):
            fn = os.path.join(where, name)
            if os.path.isdir(fn):
                bad_name = False
                for pattern in exclude_directories:
                    if (fnmatchcase(name, pattern)
                        or fn.lower() == pattern.lower()):
                        bad_name = True
                        break
                if bad_name:
                    continue
                if os.path.isfile(os.path.join(fn, '__init__.py')):
                    if not package:
                        new_package = name
                    else:
                        new_package = package + '.' + name
                        stack.append((fn, '', new_package))
                else:
                    stack.append((fn, prefix + name + os.path.sep, package))
            else:
                bad_name = False
                for pattern in exclude:
                    if (fnmatchcase(name, pattern)
                        or fn.lower() == pattern.lower()):
                        bad_name = True
                        break
                if bad_name:
                    continue
                out.setdefault(package, []).append(prefix+name)
    return out

"""
        setuppy += "setup(name='docassemble." + str(pkgname) + "',\n" + """\
      version='0.0.1',
      description=('A docassemble extension.'),
      long_description=""" + repr(readme) + """,
      long_description_content_type='text/markdown',
      author=""" + repr(str(name_of_user(current_user))) + """,
      author_email=""" + repr(str(current_user.email)) + """,
      license='MIT',
      url='https://docassemble.org',
      packages=find_packages(),
      namespace_packages = ['docassemble'],
      zip_safe = False,
      package_data=find_package_data(where=os.path.join('docassemble', '""" + str(pkgname) + """', ''), package='docassemble.""" + str(pkgname) + """'),
     )

"""
        questionfiletext = """\
---
metadata:
  title: I am the title of the application
  short title: Mobile title
  description: |
    Insert description of question file here.
  authors:
    - name: """ + str(current_user.first_name) + " " + str(current_user.last_name) + """
      organization: """ + str(current_user.organization) + """
  revision_date: """ + formatted_current_date() + """
---
mandatory: True
code: |
  user_done
---
question: |
  % if user_doing_well:
  Good to hear it!
  % else:
  Sorry to hear that!
  % endif
sets: user_done
buttons:
  - Exit: exit
  - Restart: restart
---
question: Are you doing well today?
yesno: user_doing_well
...
"""
        templatereadme = """\
# Template directory

If you want to use templates for document assembly, put them in this directory.
"""
        staticreadme = """\
# Static file directory

If you want to make files available in the web app, put them in
this directory.
"""
        sourcesreadme = """\
# Sources directory

This directory is used to store word translation files,
machine learning training files, and other source files.
"""
        objectfile = """\
# This is a Python module in which you can write your own Python code,
# if you want to.
#
# Include this module in a docassemble interview by writing:
# ---
# modules:
#   - docassemble.""" + pkgname + """.objects
# ---
#
# Then you can do things like:
# ---
# objects:
#   - favorite_fruit: Fruit
# ---
# mandatory: True
# question: |
#   When I eat some ${ favorite_fruit.name },
#   I think, "${ favorite_fruit.eat() }"
# ---
# question: What is the best fruit?
# fields:
#   - Fruit Name: favorite_fruit.name
# ---
from docassemble.base.util import DAObject

class Fruit(DAObject):
    def eat(self):
        return "Yum, that " + self.name + " was good!"
"""
        directory = tempfile.mkdtemp()
        packagedir = os.path.join(directory, 'docassemble-' + str(pkgname))
        questionsdir = os.path.join(packagedir, 'docassemble', str(pkgname), 'data', 'questions')
        templatesdir = os.path.join(packagedir, 'docassemble', str(pkgname), 'data', 'templates')
        staticdir = os.path.join(packagedir, 'docassemble', str(pkgname), 'data', 'static')
        sourcesdir = os.path.join(packagedir, 'docassemble', str(pkgname), 'data', 'sources')
        os.makedirs(questionsdir, exist_ok=True)
        os.makedirs(templatesdir, exist_ok=True)
        os.makedirs(staticdir, exist_ok=True)
        os.makedirs(sourcesdir, exist_ok=True)
        with open(os.path.join(packagedir, 'README.md'), 'w', encoding='utf-8') as the_file:
            the_file.write(readme)
        with open(os.path.join(packagedir, 'LICENSE'), 'w', encoding='utf-8') as the_file:
            the_file.write(licensetext)
        with open(os.path.join(packagedir, 'setup.py'), 'w', encoding='utf-8') as the_file:
            the_file.write(setuppy)
        with open(os.path.join(packagedir, 'setup.cfg'), 'w', encoding='utf-8') as the_file:
            the_file.write(setupcfg)
        with open(os.path.join(packagedir, 'MANIFEST.in'), 'w', encoding='utf-8') as the_file:
            the_file.write(manifestin)
        with open(os.path.join(packagedir, 'docassemble', '__init__.py'), 'w', encoding='utf-8') as the_file:
            the_file.write(initpy)
        with open(os.path.join(packagedir, 'docassemble', pkgname, '__init__.py'), 'w', encoding='utf-8') as the_file:
            the_file.write('__version__ = "0.0.1"')
        with open(os.path.join(packagedir, 'docassemble', pkgname, 'objects.py'), 'w', encoding='utf-8') as the_file:
            the_file.write(objectfile)
        with open(os.path.join(templatesdir, 'README.md'), 'w', encoding='utf-8') as the_file:
            the_file.write(templatereadme)
        with open(os.path.join(staticdir, 'README.md'), 'w', encoding='utf-8') as the_file:
            the_file.write(staticreadme)
        with open(os.path.join(sourcesdir, 'README.md'), 'w', encoding='utf-8') as the_file:
            the_file.write(sourcesreadme)
        with open(os.path.join(questionsdir, 'questions.yml'), 'w', encoding='utf-8') as the_file:
            the_file.write(questionfiletext)
        nice_name = 'docassemble-' + str(pkgname) + '.zip'
        file_number = get_new_file_number(None, nice_name)
        file_set_attributes(file_number, private=False, persistent=True)
        saved_file = SavedFile(file_number, extension='zip', fix=True, should_not_exist=True)
        zf = zipfile.ZipFile(saved_file.path, mode='w')
        trimlength = len(directory) + 1
        if current_user.timezone:
            the_timezone = pytz.timezone(current_user.timezone)
        else:
            the_timezone = pytz.timezone(get_default_timezone())
        for root, dirs, files in os.walk(packagedir):
            for the_file in files:
                thefilename = os.path.join(root, the_file)
                info = zipfile.ZipInfo(thefilename[trimlength:])
                info.date_time = datetime.datetime.utcfromtimestamp(os.path.getmtime(thefilename)).replace(tzinfo=pytz.utc).astimezone(the_timezone).timetuple()
                info.compress_type = zipfile.ZIP_DEFLATED
                info.external_attr = 0o644 << 16
                with open(thefilename, 'rb') as fp:
                    zf.writestr(info, fp.read())
                #zf.write(thefilename, thefilename[trimlength:])
        zf.close()
        saved_file.save()
        saved_file.finalize()
        shutil.rmtree(directory)
        response = send_file(saved_file.path, mimetype='application/zip', as_attachment=True, attachment_filename=nice_name)
        response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
        flash(word("Package created"), 'success')
        return response
    response = make_response(render_template('pages/create_package.html', version_warning=version_warning, bodyclass='daadminbody', form=form, tab_title=word('Create Package'), page_title=word('Create Package')), 200)
    response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
    return response

@app.route('/restart', methods=['GET', 'POST'])
@login_required
@roles_required(['admin', 'developer'])
def restart_page():
    setup_translation()
    script = """
    <script>
      function daRestartCallback(data){
        //console.log("Restart result: " + data.success);
      }
      function daRestart(){
        $.ajax({
          type: 'POST',
          url: """ + json.dumps(url_for('restart_ajax')) + """,
          data: 'csrf_token=""" + generate_csrf() + """&action=restart',
          success: daRestartCallback,
          dataType: 'json'
        });
        return true;
      }
      $( document ).ready(function() {
        //console.log("restarting");
        setTimeout(daRestart, 100);
      });
    </script>"""
    next_url = app.user_manager.make_safe_url_function(request.args.get('next', url_for('interview_list', post_restart=1)))
    extra_meta = """\n    <meta http-equiv="refresh" content="8;URL='""" + next_url + """'">"""
    response = make_response(render_template('pages/restart.html', version_warning=None, bodyclass='daadminbody', extra_meta=Markup(extra_meta), extra_js=Markup(script), tab_title=word('Restarting'), page_title=word('Restarting')), 200)
    response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
    return response

@app.route('/playground_poll', methods=['GET'])
@login_required
@roles_required(['admin', 'developer'])
def playground_poll():
    setup_translation()
    if not app.config['ENABLE_PLAYGROUND']:
        return ('File not found', 404)
    script = """
    <script>
      function daPollCallback(data){
        if (data.success){
          window.location.replace(data.url);
        }
      }
      function daPoll(){
        $.ajax({
          type: 'GET',
          url: """ + json.dumps(url_for('playground_redirect_poll')) + """,
          success: daPollCallback,
          dataType: 'json'
        });
        return true;
      }
      $( document ).ready(function() {
        //console.log("polling");
        setInterval(daPoll, 4000);
      });
    </script>"""
    response = make_response(render_template('pages/playground_poll.html', version_warning=None, bodyclass='daadminbody', extra_js=Markup(script), tab_title=word('Waiting'), page_title=word('Waiting')), 200)
    response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
    return response

def get_gd_flow():
    app_credentials = current_app.config['OAUTH_CREDENTIALS'].get('googledrive', {})
    client_id = app_credentials.get('id', None)
    client_secret = app_credentials.get('secret', None)
    if client_id is None or client_secret is None:
        raise DAError('Google Drive is not configured.')
    flow = oauth2client.client.OAuth2WebServerFlow(
        client_id=client_id,
        client_secret=client_secret,
        scope='https://www.googleapis.com/auth/drive',
        redirect_uri=url_for('google_drive_callback', _external=True),
        access_type='offline',
        prompt='consent')
    return flow

def get_gd_folder():
    key = 'da:googledrive:mapping:userid:' + str(current_user.id)
    folder = r.get(key)
    if folder is not None:
        return folder.decode()
    return folder

def set_gd_folder(folder):
    key = 'da:googledrive:mapping:userid:' + str(current_user.id)
    if folder is None:
        r.delete(key)
    else:
        set_od_folder(None)
        r.set(key, folder)

def get_od_flow():
    app_credentials = current_app.config['OAUTH_CREDENTIALS'].get('onedrive', {})
    client_id = app_credentials.get('id', None)
    client_secret = app_credentials.get('secret', None)
    if client_id is None or client_secret is None:
        raise DAError('OneDrive is not configured.')
    flow = oauth2client.client.OAuth2WebServerFlow(
        client_id=client_id,
        client_secret=client_secret,
        scope='files.readwrite.all user.read offline_access',
        redirect_uri=url_for('onedrive_callback', _external=True),
        response_type='code',
        auth_uri='https://login.microsoftonline.com/common/oauth2/v2.0/authorize',
        token_uri='https://login.microsoftonline.com/common/oauth2/v2.0/token')
    return flow

def get_od_folder():
    key = 'da:onedrive:mapping:userid:' + str(current_user.id)
    folder = r.get(key)
    if folder is not None:
        return folder.decode()
    return folder

def set_od_folder(folder):
    key = 'da:onedrive:mapping:userid:' + str(current_user.id)
    if folder is None:
        r.delete(key)
    else:
        set_gd_folder(None)
        r.set(key, folder)

class RedisCredStorage(oauth2client.client.Storage):
    def __init__(self, app='googledrive'):
        self.key = 'da:' + app + ':userid:' + str(current_user.id)
        self.lockkey = 'da:' + app + ':lock:userid:' + str(current_user.id)
    def acquire_lock(self):
        pipe = r.pipeline()
        pipe.set(self.lockkey, 1)
        pipe.expire(self.lockkey, 5)
        pipe.execute()
    def release_lock(self):
        r.delete(self.lockkey)
    def locked_get(self):
        json_creds = r.get(self.key)
        creds = None
        if json_creds is not None:
            json_creds = json_creds.decode()
            try:
                creds = oauth2client.client.Credentials.new_from_json(json_creds)
            except:
                logmessage("RedisCredStorage: could not read credentials from " + str(json_creds))
        return creds
    def locked_put(self, credentials):
        r.set(self.key, credentials.to_json())
    def locked_delete(self):
        r.delete(self.key)

@app.route('/google_drive_callback', methods=['GET', 'POST'])
@login_required
@roles_required(['admin', 'developer'])
def google_drive_callback():
    setup_translation()
    if not app.config['ENABLE_PLAYGROUND']:
        return ('File not found', 404)
    for key in request.args:
        logmessage("google_drive_callback: argument " + str(key) + ": " + str(request.args[key]))
    if 'code' in request.args:
        flow = get_gd_flow()
        credentials = flow.step2_exchange(request.args['code'])
        storage = RedisCredStorage(app='googledrive')
        storage.put(credentials)
        error = None
    elif 'error' in request.args:
        error = request.args['error']
    else:
        error = word("could not connect to Google Drive")
    if error:
        flash(word('There was a Google Drive error: ' + error), 'error')
        return redirect(url_for('user.profile'))
    else:
        flash(word('Connected to Google Drive'), 'success')
    return redirect(url_for('google_drive_page'))

def rename_gd_project(old_project, new_project):
    the_folder = get_gd_folder()
    if the_folder is None:
        logmessage('rename_gd_project: folder not configured')
        return False
    storage = RedisCredStorage(app='googledrive')
    credentials = storage.get()
    if not credentials or credentials.invalid:
        logmessage('rename_gd_project: credentials missing or expired')
        return False
    http = credentials.authorize(httplib2.Http())
    service = apiclient.discovery.build('drive', 'v3', http=http)
    response = service.files().get(fileId=the_folder, fields="mimeType, id, name, trashed").execute()
    trashed = response.get('trashed', False)
    the_mime_type = response.get('mimeType', None)
    if trashed is True or the_mime_type != "application/vnd.google-apps.folder":
        logmessage('rename_gd_project: folder did not exist')
        return False
    for section in ['static', 'templates', 'questions', 'modules', 'sources']:
        logmessage("rename_gd_project: section is " + section)
        subdir = None
        page_token = None
        while True:
            response = service.files().list(spaces="drive", pageToken=page_token, fields="nextPageToken, files(id, name)", q="mimeType='application/vnd.google-apps.folder' and trashed=false and name='" + str(section) + "' and '" + str(the_folder) + "' in parents").execute()
            for the_file in response.get('files', []):
                if 'id' in the_file:
                    subdir = the_file['id']
                    break
            page_token = response.get('nextPageToken', None)
            if subdir is not None or page_token is None:
                break
        if subdir is None:
            logmessage('rename_gd_project: section ' + str(section) + ' could not be found')
            continue
        subsubdir = None
        page_token = None
        while True:
            response = service.files().list(spaces="drive", pageToken=page_token, fields="nextPageToken, files(id, name)", q="mimeType='application/vnd.google-apps.folder' and trashed=false and name='" + str(old_project) + "' and '" + str(subdir) + "' in parents").execute()
            for the_file in response.get('files', []):
                if 'id' in the_file:
                    subsubdir = the_file['id']
                    break
            page_token = response.get('nextPageToken', None)
            if subsubdir is not None or page_token is None:
                break
        if subsubdir is None:
            logmessage('rename_gd_project: project ' + str(old_project) + ' could not be found in ' + str(section))
            continue
        metadata = {'name': new_project}
        service.files().update(fileId=subsubdir, body=metadata, fields='name').execute()
        logmessage('rename_gd_project: folder ' + str(old_project) + ' renamed in section '  + str(section))
    return True

def trash_gd_project(old_project):
    the_folder = get_gd_folder()
    if the_folder is None:
        logmessage('trash_gd_project: folder not configured')
        return False
    storage = RedisCredStorage(app='googledrive')
    credentials = storage.get()
    if not credentials or credentials.invalid:
        logmessage('trash_gd_project: credentials missing or expired')
        return False
    http = credentials.authorize(httplib2.Http())
    service = apiclient.discovery.build('drive', 'v3', http=http)
    response = service.files().get(fileId=the_folder, fields="mimeType, id, name, trashed").execute()
    trashed = response.get('trashed', False)
    the_mime_type = response.get('mimeType', None)
    if trashed is True or the_mime_type != "application/vnd.google-apps.folder":
        logmessage('trash_gd_project: folder did not exist')
        return False
    for section in ['static', 'templates', 'questions', 'modules', 'sources']:
        subdir = None
        page_token = None
        while True:
            response = service.files().list(spaces="drive", pageToken=page_token, fields="nextPageToken, files(id, name)", q="mimeType='application/vnd.google-apps.folder' and trashed=false and name='" + str(section) + "' and '" + str(the_folder) + "' in parents").execute()
            for the_file in response.get('files', []):
                if 'id' in the_file:
                    subdir = the_file['id']
                    break
            page_token = response.get('nextPageToken', None)
            if subdir is not None or page_token is None:
                break
        if subdir is None:
            logmessage('trash_gd_project: section ' + str(section) + ' could not be found')
            continue
        subsubdir = None
        page_token = None
        while True:
            response = service.files().list(spaces="drive", fields="nextPageToken, files(id, name)", q="mimeType='application/vnd.google-apps.folder' and trashed=false and name='" + str(old_project) + "' and '" + str(subdir) + "' in parents").execute()
            for the_file in response.get('files', []):
                if 'id' in the_file:
                    subsubdir = the_file['id']
                    break
            page_token = response.get('nextPageToken', None)
            if subsubdir is not None or page_token is None:
                break
        if subsubdir is None:
            logmessage('trash_gd_project: project ' + str(old_project) + ' could not be found in ' + str(section))
            continue
        service.files().delete(fileId=subsubdir).execute()
        logmessage('trash_gd_project: project ' + str(old_project) + ' deleted in section '  + str(section))
    return True

def trash_gd_file(section, filename, current_project):
    if section == 'template':
        section = 'templates'
    the_folder = get_gd_folder()
    if the_folder is None:
        logmessage('trash_gd_file: folder not configured')
        return False
    storage = RedisCredStorage(app='googledrive')
    credentials = storage.get()
    if not credentials or credentials.invalid:
        logmessage('trash_gd_file: credentials missing or expired')
        return False
    http = credentials.authorize(httplib2.Http())
    service = apiclient.discovery.build('drive', 'v3', http=http)
    response = service.files().get(fileId=the_folder, fields="mimeType, id, name, trashed").execute()
    trashed = response.get('trashed', False)
    the_mime_type = response.get('mimeType', None)
    if trashed is True or the_mime_type != "application/vnd.google-apps.folder":
        logmessage('trash_gd_file: folder did not exist')
        return False
    subdir = None
    response = service.files().list(spaces="drive", fields="nextPageToken, files(id, name)", q="mimeType='application/vnd.google-apps.folder' and trashed=false and name='" + str(section) + "' and '" + str(the_folder) + "' in parents").execute()
    for the_file in response.get('files', []):
        if 'id' in the_file:
            subdir = the_file['id']
            break
    if subdir is None:
        logmessage('trash_gd_file: section ' + str(section) + ' could not be found')
        return False
    if current_project != 'default':
        response = service.files().list(spaces="drive", fields="nextPageToken, files(id, name)", q="mimeType='application/vnd.google-apps.folder' and trashed=false and name='" + str(current_project) + "' and '" + str(subdir) + "' in parents").execute()
        subdir = None
        for the_file in response.get('files', []):
            if 'id' in the_file:
                subdir = the_file['id']
                break
        if subdir is None:
            logmessage('trash_gd_file: project ' + str(current_project) + ' could not be found')
            return False
    id_of_filename = None
    response = service.files().list(spaces="drive", fields="nextPageToken, files(id, name)", q="mimeType!='application/vnd.google-apps.folder' and name='" + str(filename) + "' and '" + str(subdir) + "' in parents").execute()
    for the_file in response.get('files', []):
        if 'id' in the_file:
            id_of_filename = the_file['id']
            break
    if id_of_filename is None:
        logmessage('trash_gd_file: file ' + str(filename) + ' could not be found in ' + str(section))
        return False
    service.files().delete(fileId=id_of_filename).execute()
    logmessage('trash_gd_file: file ' + str(filename) + ' permanently deleted from '  + str(section))
    return True

@app.route('/sync_with_google_drive', methods=['GET'])
@login_required
@roles_required(['admin', 'developer'])
def sync_with_google_drive():
    setup_translation()
    if not app.config['ENABLE_PLAYGROUND']:
        return ('File not found', 404)
    current_project = get_current_project()
    next = app.user_manager.make_safe_url_function(request.args.get('next', url_for('playground_page', project=current_project)))
    auto_next = request.args.get('auto_next', None)
    if app.config['USE_GOOGLE_DRIVE'] is False:
        flash(word("Google Drive is not configured"), "error")
        return redirect(next)
    storage = RedisCredStorage(app='googledrive')
    credentials = storage.get()
    if not credentials or credentials.invalid:
        flow = get_gd_flow()
        uri = flow.step1_get_authorize_url()
        return redirect(uri)
    task = docassemble.webapp.worker.sync_with_google_drive.delay(current_user.id)
    session['taskwait'] = task.id
    if auto_next:
        return redirect(url_for('gd_sync_wait', auto_next=auto_next))
    else:
        return redirect(url_for('gd_sync_wait', next=next))

@app.route('/gdsyncing', methods=['GET', 'POST'])
@login_required
@roles_required(['admin', 'developer'])
def gd_sync_wait():
    setup_translation()
    if not app.config['ENABLE_PLAYGROUND']:
        return ('File not found', 404)
    current_project = get_current_project()
    next_url = app.user_manager.make_safe_url_function(request.args.get('next', url_for('playground_page', project=current_project)))
    auto_next_url = request.args.get('auto_next', None)
    my_csrf = generate_csrf()
    script = """
    <script>
      var daCheckinInterval = null;
      var autoNext = """ + json.dumps(auto_next_url) + """;
      var resultsAreIn = false;
      function daRestartCallback(data){
        //console.log("Restart result: " + data.success);
      }
      function daRestart(){
        $.ajax({
          type: 'POST',
          url: """ + json.dumps(url_for('restart_ajax')) + """,
          data: 'csrf_token=""" + my_csrf + """&action=restart',
          success: daRestartCallback,
          dataType: 'json'
        });
        return true;
      }
      function daSyncCallback(data){
        if (data.success){
          if (data.status == 'finished'){
            resultsAreIn = true;
            if (data.ok){
              if (autoNext != null){
                window.location.replace(autoNext);
              }
              $("#notification").html(""" + json.dumps(word("The synchronization was successful.")) + """);
              $("#notification").removeClass("alert-info");
              $("#notification").removeClass("alert-danger");
              $("#notification").addClass("alert-success");
            }
            else{
              $("#notification").html(""" + json.dumps(word("The synchronization was not successful.")) + """);
              $("#notification").removeClass("alert-info");
              $("#notification").removeClass("alert-success");
              $("#notification").addClass("alert-danger");
            }
            $("#resultsContainer").show();
            $("#resultsArea").html(data.summary);
            if (daCheckinInterval != null){
              clearInterval(daCheckinInterval);
            }
            if (data.restart){
              daRestart();
            }
          }
          else if (data.status == 'failed' && !resultsAreIn){
            resultsAreIn = true;
            $("#notification").html(""" + json.dumps(word("There was an error with the synchronization.")) + """);
            $("#notification").removeClass("alert-info");
            $("#notification").removeClass("alert-success");
            $("#notification").addClass("alert-danger");
            $("#resultsContainer").show();
            if (data.error_message){
              $("#resultsArea").html(data.error_message);
            }
            else if (data.summary){
              $("#resultsArea").html(data.summary);
            }
            if (daCheckinInterval != null){
              clearInterval(daCheckinInterval);
            }
          }
        }
        else if (!resultsAreIn){
          $("#notification").html(""" + json.dumps(word("There was an error.")) + """);
          $("#notification").removeClass("alert-info");
          $("#notification").removeClass("alert-success");
          $("#notification").addClass("alert-danger");
          if (daCheckinInterval != null){
            clearInterval(daCheckinInterval);
          }
        }
      }
      function daSync(){
        if (resultsAreIn){
          return;
        }
        $.ajax({
          type: 'POST',
          url: """ + json.dumps(url_for('checkin_sync_with_google_drive')) + """,
          data: 'csrf_token=""" + my_csrf + """',
          success: daSyncCallback,
          dataType: 'json'
        });
        return true;
      }
      $( document ).ready(function() {
        //console.log("page loaded");
        daCheckinInterval = setInterval(daSync, 2000);
      });
    </script>"""
    response = make_response(render_template('pages/gd_sync_wait.html', version_warning=None, bodyclass='daadminbody', extra_js=Markup(script), tab_title=word('Synchronizing'), page_title=word('Synchronizing'), next_page=next_url), 200)
    response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
    return response

@app.route('/onedrive_callback', methods=['GET', 'POST'])
@login_required
@roles_required(['admin', 'developer'])
def onedrive_callback():
    setup_translation()
    if not app.config['ENABLE_PLAYGROUND']:
        return ('File not found', 404)
    for key in request.args:
        logmessage("onedrive_callback: argument " + str(key) + ": " + str(request.args[key]))
    if 'code' in request.args:
        flow = get_od_flow()
        credentials = flow.step2_exchange(request.args['code'])
        storage = RedisCredStorage(app='onedrive')
        storage.put(credentials)
        error = None
    elif 'error' in request.args:
        error = request.args['error']
        if 'error_description' in request.args:
            error += '; ' + str(request.args['error_description'])
    else:
        error = word("could not connect to OneDrive")
    if error:
        flash(word('There was a OneDrive error: ' + error), 'error')
        return redirect(url_for('user.profile'))
    else:
        flash(word('Connected to OneDrive'), 'success')
    return redirect(url_for('onedrive_page'))

def rename_od_project(old_project, new_project):
    the_folder = get_od_folder()
    if the_folder is None:
        logmessage('rename_od_project: folder not configured')
        return False
    storage = RedisCredStorage(app='onedrive')
    credentials = storage.get()
    if not credentials or credentials.invalid:
        logmessage('rename_od_project: credentials missing or expired')
        return False
    http = credentials.authorize(httplib2.Http())
    r, content = http.request("https://graph.microsoft.com/v1.0/me/drive/items/" + urllibquote(the_folder), "GET")
    if int(r['status']) != 200:
        trashed = True
    else:
        info = json.loads(content.decode())
        #logmessage("Found " + repr(info))
        trashed = bool(info.get('deleted', None))
    if trashed is True or 'folder' not in info:
        logmessage('rename_od_project: folder did not exist')
        return False
    r, content = http.request("https://graph.microsoft.com/v1.0/me/drive/items/" + urllibquote(the_folder) + "/children?$select=id,name,deleted,folder", "GET")
    subdir = {}
    for section in ['static', 'templates', 'questions', 'modules', 'sources']:
        subdir[section] = None
    while True:
        if int(r['status']) != 200:
            logmessage('rename_od_project: could not obtain subfolders')
            return False
        info = json.loads(content.decode())
        for item in info.get('value', []):
            if item.get('deleted', None) or 'folder' not in item:
                continue
            if item['name'] in subdir:
                subdir[item['name']] = item['id']
        if "@odata.nextLink" not in info:
            break
        r, content = http.request(info["@odata.nextLink"], "GET")
    for section in subdir.keys():
        if subdir[section] is None:
            logmessage('rename_od_project: could not obtain subfolder for ' + str(section))
            continue
        subsubdir = None
        r, content = http.request("https://graph.microsoft.com/v1.0/me/drive/items/" + str(subdir[section]) + "/children?$select=id,name,deleted,folder", "GET")
        while True:
            if int(r['status']) != 200:
                logmessage('rename_od_project: could not obtain contents of subfolder for ' + str(section))
                break
            info = json.loads(content.decode())
            for item in info.get('value', []):
                if item.get('deleted', None) or 'folder' not in item:
                    continue
                if item['name'] == old_project:
                    subsubdir = item['id']
                    break
            if subsubdir is not None or "@odata.nextLink" not in info:
                break
            r, content = http.request(info["@odata.nextLink"], "GET")
        if subsubdir is None:
            logmessage("rename_od_project: subdirectory " + str(old_project) + " not found")
        else:
            headers = { 'Content-Type': 'application/json' }
            r, content = http.request("https://graph.microsoft.com/v1.0/me/drive/items/" + str(subsubdir), "PATCH", headers=headers, body=json.dumps(dict(name=new_project)))
            if int(r['status']) != 200:
                logmessage('rename_od_project: could not rename folder ' + str(old_project) + " in " + str(section) + " because " + repr(content))
                continue
        logmessage('rename_od_project: project ' + str(old_project) + ' rename in section '  + str(section))
    return True

def trash_od_project(old_project):
    the_folder = get_od_folder()
    if the_folder is None:
        logmessage('trash_od_project: folder not configured')
        return False
    storage = RedisCredStorage(app='onedrive')
    credentials = storage.get()
    if not credentials or credentials.invalid:
        logmessage('trash_od_project: credentials missing or expired')
        return False
    http = credentials.authorize(httplib2.Http())
    r, content = http.request("https://graph.microsoft.com/v1.0/me/drive/items/" + urllibquote(the_folder), "GET")
    if int(r['status']) != 200:
        trashed = True
    else:
        info = json.loads(content.decode())
        #logmessage("Found " + repr(info))
        trashed = bool(info.get('deleted', None))
    if trashed is True or 'folder' not in info:
        logmessage('trash_od_project: folder did not exist')
        return False
    subdir = {}
    for section in ['static', 'templates', 'questions', 'modules', 'sources']:
        subdir[section] = None
    r, content = http.request("https://graph.microsoft.com/v1.0/me/drive/items/" + urllibquote(the_folder) + "/children?$select=id,name,deleted,folder", "GET")
    while True:
        if int(r['status']) != 200:
            logmessage('trash_od_project: could not obtain subfolders')
            return False
        info = json.loads(content.decode())
        for item in info['value']:
            if item.get('deleted', None) or 'folder' not in item:
                continue
            if item['name'] in subdir:
                subdir[item['name']] = item['id']
        if "@odata.nextLink" not in info:
            break
        r, content = http.request(info["@odata.nextLink"], "GET")
    for section in subdir.keys():
        if subdir[section] is None:
            logmessage('trash_od_project: could not obtain subfolder for ' + str(section))
            continue
        subsubdir = None
        r, content = http.request("https://graph.microsoft.com/v1.0/me/drive/items/" + str(subdir[section]) + "/children?$select=id,name,deleted,folder", "GET")
        while True:
            if int(r['status']) != 200:
                logmessage('trash_od_project: could not obtain contents of subfolder for ' + str(section))
                break
            info = json.loads(content.decode())
            for item in info['value']:
                if item.get('deleted', None) or 'folder' not in item:
                    continue
                if item['name'] == old_project:
                    subsubdir = item['id']
                    break
            if subsubdir is not None or "@odata.nextLink" not in info:
                break
            r, content = http.request(info["@odata.nextLink"], "GET")
        if subsubdir is None:
            logmessage("Could not find subdirectory " + old_project + " in section " + str(section))
        else:
            r, content = http.request("https://graph.microsoft.com/v1.0/me/drive/items/" + urllibquote(subsubdir) + "/children?$select=id", "GET")
            to_delete = []
            while True:
                if int(r['status']) != 200:
                    logmessage('trash_od_project: could not obtain contents of project folder')
                    return False
                info = json.loads(content.decode())
                for item in info.get('value', []):
                    if 'id' in item:
                        to_delete.append(item['id'])
                if "@odata.nextLink" not in info:
                    break
                r, content = http.request(info["@odata.nextLink"], "GET")
            for item_id in to_delete:
                r, content = http.request("https://graph.microsoft.com/v1.0/me/drive/items/" + str(item_id), "DELETE")
                if int(r['status']) != 204:
                    logmessage('trash_od_project: could not delete file ' + str(item_id) + ".  Result: " + repr(content))
                    return False
            r, content = http.request("https://graph.microsoft.com/v1.0/me/drive/items/" + str(subsubdir), "DELETE")
            if int(r['status']) != 204:
                logmessage('trash_od_project: could not delete project ' + str(old_project) + ".  Result: " + repr(content))
                return False
            logmessage('trash_od_project: project ' + str(old_project) + ' trashed in section '  + str(section))
    return True

def trash_od_file(section, filename, current_project):
    if section == 'template':
        section = 'templates'
    the_folder = get_od_folder()
    if the_folder is None:
        logmessage('trash_od_file: folder not configured')
        return False
    storage = RedisCredStorage(app='onedrive')
    credentials = storage.get()
    if not credentials or credentials.invalid:
        logmessage('trash_od_file: credentials missing or expired')
        return False
    http = credentials.authorize(httplib2.Http())
    r, content = http.request("https://graph.microsoft.com/v1.0/me/drive/items/" + urllibquote(the_folder), "GET")
    if int(r['status']) != 200:
        trashed = True
    else:
        info = json.loads(content.decode())
        #logmessage("Found " + repr(info))
        trashed = bool(info.get('deleted', None))
    if trashed is True or 'folder' not in info:
        logmessage('trash_od_file: folder did not exist')
        return False
    r, content = http.request("https://graph.microsoft.com/v1.0/me/drive/items/" + urllibquote(the_folder) + "/children?$select=id,name,deleted,folder", "GET")
    subdir = None
    while True:
        if int(r['status']) != 200:
            logmessage('trash_od_file: could not obtain subfolders')
            return False
        info = json.loads(content.decode())
        #logmessage("Found " + repr(info))
        for item in info['value']:
            if item.get('deleted', None) or 'folder' not in item:
                continue
            if item['name'] == section:
                subdir = item['id']
                break
        if subdir is not None or "@odata.nextLink" not in info:
            break
        r, content = http.request(info["@odata.nextLink"], "GET")
    if subdir is None:
        logmessage('trash_od_file: could not obtain subfolder')
        return False
    if current_project != 'default':
        r, content = http.request("https://graph.microsoft.com/v1.0/me/drive/items/" + str(subdir) + "/children?$select=id,name,deleted,folder", "GET")
        subdir = None
        while True:
            if int(r['status']) != 200:
                logmessage('trash_od_file: could not obtain subfolders to find project')
                return False
            info = json.loads(content.decode())
            for item in info['value']:
                if item.get('deleted', None) or 'folder' not in item:
                    continue
                if item['name'] == current_project:
                    subdir = item['id']
                    break
            if subdir is not None or "@odata.nextLink" not in info:
                break
            r, content = http.request(info["@odata.nextLink"], "GET")
        if subdir is None:
            logmessage('trash_od_file: could not obtain subfolder')
            return False
    id_of_filename = None
    r, content = http.request("https://graph.microsoft.com/v1.0/me/drive/items/" + str(subdir) + "/children?$select=id,name,deleted,folder", "GET")
    while True:
        if int(r['status']) != 200:
            logmessage('trash_od_file: could not obtain contents of subfolder')
            return False
        info = json.loads(content.decode())
        #logmessage("Found " + repr(info))
        for item in info['value']:
            if item.get('deleted', None) or 'folder' in item:
                continue
            if 'folder' in item:
                continue
            if item['name'] == filename:
                id_of_filename = item['id']
                break
        if id_of_filename is not None or "@odata.nextLink" not in info:
            break
        r, content = http.request(info["@odata.nextLink"], "GET")
    r, content = http.request("https://graph.microsoft.com/v1.0/me/drive/items/" + str(id_of_filename), "DELETE")
    if int(r['status']) != 204:
        logmessage('trash_od_file: could not delete ')
        return False
    logmessage('trash_od_file: file ' + str(filename) + ' trashed from '  + str(section))
    return True

@app.route('/sync_with_onedrive', methods=['GET'])
@login_required
@roles_required(['admin', 'developer'])
def sync_with_onedrive():
    setup_translation()
    if not app.config['ENABLE_PLAYGROUND']:
        return ('File not found', 404)
    current_project = get_current_project()
    next = app.user_manager.make_safe_url_function(request.args.get('next', url_for('playground_page', project=get_current_project())))
    auto_next = request.args.get('auto_next', None)
    if app.config['USE_ONEDRIVE'] is False:
        flash(word("OneDrive is not configured"), "error")
        return redirect(next)
    storage = RedisCredStorage(app='onedrive')
    credentials = storage.get()
    if not credentials or credentials.invalid:
        flow = get_gd_flow()
        uri = flow.step1_get_authorize_url()
        return redirect(uri)
    task = docassemble.webapp.worker.sync_with_onedrive.delay(current_user.id)
    session['taskwait'] = task.id
    if auto_next:
        return redirect(url_for('od_sync_wait', auto_next=auto_next))
    else:
        return redirect(url_for('od_sync_wait', next=next))

@app.route('/odsyncing', methods=['GET', 'POST'])
@login_required
@roles_required(['admin', 'developer'])
def od_sync_wait():
    setup_translation()
    if not app.config['ENABLE_PLAYGROUND']:
        return ('File not found', 404)
    current_project = get_current_project()
    next_url = app.user_manager.make_safe_url_function(request.args.get('next', url_for('playground_page', project=current_project)))
    auto_next_url = request.args.get('auto_next', None)
    if auto_next_url is not None:
        auto_next_url = app.user_manager.make_safe_url_function(auto_next_url)
    my_csrf = generate_csrf()
    script = """
    <script>
      var daCheckinInterval = null;
      var autoNext = """ + json.dumps(auto_next_url) + """;
      var resultsAreIn = false;
      function daRestartCallback(data){
        if (autoNext != null){
          setTimeout(function(){
            window.location.replace(autoNext);
          }, 1000);
        }
        //console.log("Restart result: " + data.success);
      }
      function daRestart(){
        $.ajax({
          type: 'POST',
          url: """ + json.dumps(url_for('restart_ajax')) + """,
          data: 'csrf_token=""" + my_csrf + """&action=restart',
          success: daRestartCallback,
          dataType: 'json'
        });
        return true;
      }
      function daSyncCallback(data){
        if (data.success){
          if (data.status == 'finished'){
            resultsAreIn = true;
            if (data.ok){
              $("#notification").html(""" + json.dumps(word("The synchronization was successful.")) + """);
              $("#notification").removeClass("alert-info");
              $("#notification").removeClass("alert-danger");
              $("#notification").addClass("alert-success");
            }
            else{
              $("#notification").html(""" + json.dumps(word("The synchronization was not successful.")) + """);
              $("#notification").removeClass("alert-info");
              $("#notification").removeClass("alert-success");
              $("#notification").addClass("alert-danger");
            }
            $("#resultsContainer").show();
            $("#resultsArea").html(data.summary);
            if (daCheckinInterval != null){
              clearInterval(daCheckinInterval);
            }
            if (data.restart){
              daRestart();
            }
            else{
              if (autoNext != null){
                window.location.replace(autoNext);
              }
            }
          }
          else if (data.status == 'failed' && !resultsAreIn){
            resultsAreIn = true;
            $("#notification").html(""" + json.dumps(word("There was an error with the synchronization.")) + """);
            $("#notification").removeClass("alert-info");
            $("#notification").removeClass("alert-success");
            $("#notification").addClass("alert-danger");
            $("#resultsContainer").show();
            if (data.error_message){
              $("#resultsArea").html(data.error_message);
            }
            else if (data.summary){
              $("#resultsArea").html(data.summary);
            }
            if (daCheckinInterval != null){
              clearInterval(daCheckinInterval);
            }
          }
        }
        else if (!resultsAreIn){
          $("#notification").html(""" + json.dumps(word("There was an error.")) + """);
          $("#notification").removeClass("alert-info");
          $("#notification").removeClass("alert-success");
          $("#notification").addClass("alert-danger");
          if (daCheckinInterval != null){
            clearInterval(daCheckinInterval);
          }
        }
      }
      function daSync(){
        if (resultsAreIn){
          return;
        }
        $.ajax({
          type: 'POST',
          url: """ + json.dumps(url_for('checkin_sync_with_onedrive')) + """,
          data: 'csrf_token=""" + my_csrf + """',
          success: daSyncCallback,
          dataType: 'json'
        });
        return true;
      }
      $( document ).ready(function() {
        //console.log("page loaded");
        daCheckinInterval = setInterval(daSync, 2000);
      });
    </script>"""
    response = make_response(render_template('pages/od_sync_wait.html', version_warning=None, bodyclass='daadminbody', extra_js=Markup(script), tab_title=word('Synchronizing'), page_title=word('Synchronizing'), next_page=next_url), 200)
    response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
    return response

# @app.route('/old_sync_with_google_drive', methods=['GET', 'POST'])
# @login_required
# @roles_required(['admin', 'developer'])
# def old_sync_with_google_drive():
#     next = request.args.get('next', url_for('playground_page'))
#     extra_meta = """\n    <meta http-equiv="refresh" content="1; url='""" + url_for('do_sync_with_google_drive', next=next) + """'">"""
#     return render_template('pages/google_sync.html', version_warning=None, bodyclass='daadminbody', extra_meta=Markup(extra_meta), tab_title=word('Synchronizing'), page_title=word('Synchronizing'))

def add_br(text):
    return re.sub(r'[\n\r]+', "<br>", text)

@app.route('/checkin_sync_with_google_drive', methods=['GET', 'POST'])
@login_required
@roles_required(['admin', 'developer'])
def checkin_sync_with_google_drive():
    if not app.config['ENABLE_PLAYGROUND']:
        return ('File not found', 404)
    setup_translation()
    if 'taskwait' not in session:
        return jsonify(success=False)
    result = docassemble.webapp.worker.workerapp.AsyncResult(id=session['taskwait'])
    if result.ready():
        if 'taskwait' in session:
            del session['taskwait']
        the_result = result.get()
        if isinstance(the_result, ReturnValue):
            if the_result.ok:
                logmessage("checkin_sync_with_google_drive: success")
                return jsonify(success=True, status='finished', ok=the_result.ok, summary=add_br(the_result.summary), restart=the_result.restart)
            if hasattr(the_result, 'error'):
                logmessage("checkin_sync_with_google_drive: failed return value is " + str(the_result.error))
                return jsonify(success=True, status='failed', error_message=str(the_result.error), restart=False)
            if hasattr(the_result, 'summary'):
                return jsonify(success=True, status='failed', summary=add_br(the_result.summary), restart=False)
            return jsonify(success=True, status='failed', error_message=str("No error message.  Result is " + str(the_result)), restart=False)
        else:
            logmessage("checkin_sync_with_google_drive: failed return value is a " + str(type(the_result)))
            logmessage("checkin_sync_with_google_drive: failed return value is " + str(the_result))
            return jsonify(success=True, status='failed', error_message=noquote(str(the_result)), restart=False)
    else:
        return jsonify(success=True, status='waiting', restart=False)

@app.route('/checkin_sync_with_onedrive', methods=['GET', 'POST'])
@login_required
@roles_required(['admin', 'developer'])
def checkin_sync_with_onedrive():
    if not app.config['ENABLE_PLAYGROUND']:
        return ('File not found', 404)
    setup_translation()
    if 'taskwait' not in session:
        return jsonify(success=False)
    result = docassemble.webapp.worker.workerapp.AsyncResult(id=session['taskwait'])
    if result.ready():
        if 'taskwait' in session:
            del session['taskwait']
        the_result = result.get()
        if isinstance(the_result, ReturnValue):
            if the_result.ok:
                logmessage("checkin_sync_with_onedrive: success")
                return jsonify(success=True, status='finished', ok=the_result.ok, summary=add_br(the_result.summary), restart=the_result.restart)
            if hasattr(the_result, 'error'):
                logmessage("checkin_sync_with_onedrive: failed return value is " + str(the_result.error))
                return jsonify(success=True, status='failed', error_message=str(the_result.error), restart=False)
            if hasattr(the_result, 'summary'):
                return jsonify(success=True, status='failed', summary=add_br(the_result.summary), restart=False)
            return jsonify(success=True, status='failed', error_message=str("No error message.  Result is " + str(the_result)), restart=False)
        logmessage("checkin_sync_with_onedrive: failed return value is a " + str(type(the_result)))
        logmessage("checkin_sync_with_onedrive: failed return value is " + str(the_result))
        return jsonify(success=True, status='failed', error_message=str(the_result), restart=False)
    return jsonify(success=True, status='waiting', restart=False)

@app.route('/google_drive', methods=['GET', 'POST'])
@login_required
@roles_required(['admin', 'developer'])
def google_drive_page():
    setup_translation()
    if not app.config['ENABLE_PLAYGROUND']:
        return ('File not found', 404)
    if app.config['USE_GOOGLE_DRIVE'] is False:
        flash(word("Google Drive is not configured"), "error")
        return redirect(url_for('user.profile'))
    form = GoogleDriveForm(request.form)
    if request.method == 'POST' and form.cancel.data:
        return redirect(url_for('user.profile'))
    storage = RedisCredStorage(app='googledrive')
    credentials = storage.get()
    if not credentials or credentials.invalid:
        flow = get_gd_flow()
        uri = flow.step1_get_authorize_url()
        # logmessage("google_drive_page: uri is " + str(uri))
        return redirect(uri)
    http = credentials.authorize(httplib2.Http())
    try:
        service = apiclient.discovery.build('drive', 'v3', http=http)
    except:
        set_gd_folder(None)
        storage.release_lock()
        storage.locked_delete()
        flow = get_gd_flow()
        uri = flow.step1_get_authorize_url()
        return redirect(uri)
    items = [dict(id='', name=word('-- Do not link --'))]
    #items = []
    page_token = None
    while True:
        try:
            response = service.files().list(spaces="drive", pageToken=page_token, fields="nextPageToken, files(id, name, mimeType, shortcutDetails)", q="trashed=false and 'root' in parents and (mimeType = 'application/vnd.google-apps.folder' or (mimeType = 'application/vnd.google-apps.shortcut' and shortcutDetails.targetMimeType = 'application/vnd.google-apps.folder'))").execute()
        except Exception as err:
            logmessage("google_drive_page: " + err.__class__.__name__ + ": " + str(err))
            set_gd_folder(None)
            storage.release_lock()
            storage.locked_delete()
            flash(word('There was a Google Drive error: ' + err.__class__.__name__ + ": " + str(err)), 'error')
            return redirect(url_for('google_drive_page'))
        for the_file in response.get('files', []):
            if the_file['mimeType'] == 'application/vnd.google-apps.shortcut':
                the_file['id'] = the_file['shortcutDetails']['targetId']
            items.append(the_file)
        page_token = response.get('nextPageToken', None)
        if page_token is None:
            break
    item_ids = [x['id'] for x in items if x['id'] != '']
    if request.method == 'POST' and form.submit.data:
        if form.folder.data == '':
            set_gd_folder(None)
            storage.locked_delete()
            flash(word("Google Drive is not linked."), 'success')
        elif form.folder.data == -1 or form.folder.data == '-1':
            file_metadata = {
                'name' : 'docassemble',
                'mimeType' : 'application/vnd.google-apps.folder'
            }
            new_file = service.files().create(body=file_metadata,
                                              fields='id').execute()
            new_folder = new_file.get('id', None)
            set_gd_folder(new_folder)
            gd_fix_subdirs(service, new_folder)
            if new_folder is not None:
                active_folder = dict(id=new_folder, name='docassemble')
                items.append(active_folder)
                item_ids.append(new_folder)
            flash(word("Your Playground is connected to your Google Drive."), 'success')
        elif form.folder.data in item_ids:
            flash(word("Your Playground is connected to your Google Drive."), 'success')
            set_gd_folder(form.folder.data)
            gd_fix_subdirs(service, form.folder.data)
        else:
            flash(word("The supplied folder " + str(form.folder.data) + "could not be found."), 'error')
            set_gd_folder(None)
        return redirect(url_for('user.profile'))
    the_folder = get_gd_folder()
    active_folder = None
    if the_folder is not None:
        try:
            response = service.files().get(fileId=the_folder, fields="mimeType, trashed").execute()
        except:
            set_gd_folder(None)
            return redirect(url_for('google_drive_page'))
        the_mime_type = response.get('mimeType', None)
        trashed = response.get('trashed', False)
        if trashed is False and the_mime_type == "application/vnd.google-apps.folder":
            active_folder = dict(id=the_folder, name=response.get('name', 'no name'))
            if the_folder not in item_ids:
                items.append(active_folder)
        else:
            set_gd_folder(None)
            the_folder = None
            flash(word("The mapping was reset because the folder does not appear to exist anymore."), 'error')
    if the_folder is None:
        for item in items:
            if item['name'].lower() == 'docassemble':
                active_folder = item
                break
    if active_folder is None:
        active_folder = dict(id=-1, name='docassemble')
        items.append(active_folder)
        item_ids.append(-1)
    if the_folder is not None:
        gd_fix_subdirs(service, the_folder)
    if the_folder is None:
        the_folder = ''
    description = 'Select the folder from your Google Drive that you want to be synchronized with the Playground.'
    if app.config['USE_ONEDRIVE'] is True and get_od_folder() is not None:
        description += '  ' + word('Note that if you connect to a Google Drive folder, you will disable your connection to OneDrive.')

    response = make_response(render_template('pages/googledrive.html', version_warning=version_warning, description=description, bodyclass='daadminbody', header=word('Google Drive'), tab_title=word('Google Drive'), items=items, the_folder=the_folder, page_title=word('Google Drive'), form=form), 200)
    response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
    return response

def gd_fix_subdirs(service, the_folder):
    subdirs = []
    page_token = None
    while True:
        response = service.files().list(spaces="drive", pageToken=page_token, fields="nextPageToken, files(id, name)", q="mimeType='application/vnd.google-apps.folder' and trashed=false and '" + str(the_folder) + "' in parents").execute()
        for the_file in response.get('files', []):
            subdirs.append(the_file)
        page_token = response.get('nextPageToken', None)
        if page_token is None:
            break
    todo = set(['questions', 'static', 'sources', 'templates', 'modules'])
    done = set(x['name'] for x in subdirs if x['name'] in todo)
    for key in todo - done:
        file_metadata = {
            'name' : key,
            'mimeType' : 'application/vnd.google-apps.folder',
            'parents': [the_folder]
        }
        new_file = service.files().create(body=file_metadata,
                                          fields='id').execute()

@app.route('/onedrive', methods=['GET', 'POST'])
@login_required
@roles_required(['admin', 'developer'])
def onedrive_page():
    setup_translation()
    if not app.config['ENABLE_PLAYGROUND']:
        return ('File not found', 404)
    if app.config['USE_ONEDRIVE'] is False:
        flash(word("OneDrive is not configured"), "error")
        return redirect(url_for('user.profile'))
    form = OneDriveForm(request.form)
    if request.method == 'POST' and form.cancel.data:
        return redirect(url_for('user.profile'))
    storage = RedisCredStorage(app='onedrive')
    credentials = storage.get()
    if not credentials or credentials.invalid:
        flow = get_od_flow()
        uri = flow.step1_get_authorize_url()
        logmessage("one_drive_page: uri is " + str(uri))
        return redirect(uri)
    items = [dict(id='', name=word('-- Do not link --'))]
    http = credentials.authorize(httplib2.Http())
    try:
        r, content = http.request("https://graph.microsoft.com/v1.0/me/drive/root/children?$select=id,name,deleted,folder", "GET")
    except Exception as err:
        set_od_folder(None)
        storage.release_lock()
        storage.locked_delete()
        flow = get_od_flow()
        uri = flow.step1_get_authorize_url()
        logmessage("one_drive_page: uri is " + str(uri))
        return redirect(uri)
    while True:
        if int(r['status']) != 200:
            flash("Error: could not connect to OneDrive; response code was " + str(r['status']) + ".   " + content.decode(), 'danger')
            return redirect(url_for('user.profile'))
        info = json.loads(content.decode())
        for item in info['value']:
            if 'folder' not in item:
                continue
            items.append(dict(id=item['id'], name=item['name']))
        if "@odata.nextLink" not in info:
            break
        r, content = http.request(info["@odata.nextLink"], "GET")
    item_ids = [x['id'] for x in items if x['id'] != '']
    if request.method == 'POST' and form.submit.data:
        if form.folder.data == '':
            set_od_folder(None)
            storage.locked_delete()
            flash(word("OneDrive is not linked."), 'success')
        elif form.folder.data == -1 or form.folder.data == '-1':
            headers = {'Content-Type': 'application/json'}
            info = {}
            info['name'] = 'docassemble'
            info['folder'] = {}
            info["@microsoft.graph.conflictBehavior"] = "fail"
            r, content = http.request("https://graph.microsoft.com/v1.0/me/drive/root/children", "POST", headers=headers, body=json.dumps(info))
            if int(r['status']) == 201:
                new_item = json.loads(content.decode())
                set_od_folder(new_item['id'])
                od_fix_subdirs(http, new_item['id'])
                flash(word("Your Playground is connected to your OneDrive."), 'success')
            else:
                flash(word("Could not create folder.  " + content.decode()), 'danger')
        elif form.folder.data in item_ids:
            set_od_folder(form.folder.data)
            od_fix_subdirs(http, form.folder.data)
            flash(word("Your Playground is connected to your OneDrive."), 'success')
        else:
            flash(word("The supplied folder " + str(form.folder.data) + "could not be found."), 'danger')
            set_od_folder(None)
        return redirect(url_for('user.profile'))
    the_folder = get_od_folder()
    active_folder = None
    if the_folder is not None:
        r, content = http.request("https://graph.microsoft.com/v1.0/me/drive/items/" + str(the_folder), "GET")
        if int(r['status']) != 200:
            set_od_folder(None)
            flash(word("The previously selected OneDrive folder does not exist.") + "  " + str(the_folder) + " " + str(content) + " status: " + repr(r['status']), "info")
            return redirect(url_for('onedrive_page'))
        info = json.loads(content.decode())
        logmessage("Found " + repr(info))
        if info.get('deleted', None):
            set_od_folder(None)
            flash(word("The previously selected OneDrive folder was deleted."), "info")
            return redirect(url_for('onedrive_page'))
        active_folder = dict(id=the_folder, name=info.get('name', 'no name'))
        if the_folder not in item_ids:
            items.append(active_folder)
            item_ids.append(the_folder)
    if the_folder is None:
        for item in items:
            if item['name'].lower() == 'docassemble':
                active_folder = item
                break
    if active_folder is None:
        active_folder = dict(id=-1, name='docassemble')
        items.append(active_folder)
        item_ids.append(-1)
    if the_folder is not None:
        od_fix_subdirs(http, the_folder)
    if the_folder is None:
        the_folder = ''
    description = word('Select the folder from your OneDrive that you want to be synchronized with the Playground.')
    if app.config['USE_GOOGLE_DRIVE'] is True and get_gd_folder() is not None:
        description += '  ' + word('Note that if you connect to a OneDrive folder, you will disable your connection to Google Drive.')
    response = make_response(render_template('pages/onedrive.html', version_warning=version_warning, bodyclass='daadminbody', header=word('OneDrive'), tab_title=word('OneDrive'), items=items, the_folder=the_folder, page_title=word('OneDrive'), form=form, description=Markup(description)), 200)
    response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
    return response

def od_fix_subdirs(http, the_folder):
    subdirs = set()
    r, content = http.request("https://graph.microsoft.com/v1.0/me/drive/items/" + str(the_folder) + "/children?$select=id,name,deleted,folder", "GET")
    while True:
        if int(r['status']) != 200:
            raise DAError("od_fix_subdirs: could not get contents of folder")
        info = json.loads(content.decode())
        logmessage("Found " + repr(info))
        for item in info['value']:
            if 'folder' in item:
                subdirs.add(item['name'])
        if "@odata.nextLink" not in info:
            break
        r, content = http.request(info["@odata.nextLink"], "GET")
    todo = set(['questions', 'static', 'sources', 'templates', 'modules'])
    for folder_name in (todo - subdirs):
        headers = {'Content-Type': 'application/json'}
        data = {}
        data['name'] = folder_name
        data['folder'] = {}
        data["@microsoft.graph.conflictBehavior"] = "rename"
        r, content = http.request("https://graph.microsoft.com/v1.0/me/drive/items/" + str(the_folder) + "/children", "POST", headers=headers, body=json.dumps(data))
        if int(r['status']) != 201:
            raise DAError("od_fix_subdirs: could not create subfolder " + folder_name + ' in ' + str(the_folder) + '.  ' + content.decode() + ' status: ' + str(r['status']))

@app.route('/config', methods=['GET', 'POST'])
@login_required
@roles_required(['admin'])
def config_page():
    setup_translation()
    form = ConfigForm(request.form)
    content = None
    ok = True
    if request.method == 'POST':
        if form.submit.data and form.config_content.data:
            try:
                yaml.load(form.config_content.data, Loader=yaml.FullLoader)
            except Exception as errMess:
                ok = False
                content = form.config_content.data
                errMess = word("Configuration not updated.  There was a syntax error in the configuration YAML.") + '<pre>' + str(errMess) + '</pre>'
                flash(str(errMess), 'error')
                logmessage('config_page: ' + str(errMess))
            if ok:
                if cloud is not None:
                    key = cloud.get_key('config.yml')
                    key.set_contents_from_string(form.config_content.data)
                with open(daconfig['config file'], 'w', encoding='utf-8') as fp:
                    fp.write(form.config_content.data)
                    flash(word('The configuration file was saved.'), 'success')
                #session['restart'] = 1
                return redirect(url_for('restart_page'))
        elif form.cancel.data:
            flash(word('Configuration not updated.'), 'info')
            return redirect(url_for('interview_list'))
        else:
            flash(word('Configuration not updated.  There was an error.'), 'error')
            return redirect(url_for('interview_list'))
    if ok:
        with open(daconfig['config file'], 'r', encoding='utf-8') as fp:
            content = fp.read()
    if content is None:
        return ('File not found', 404)
    (disk_total, disk_used, disk_free) = shutil.disk_usage(daconfig['config file'])
    if keymap:
        kbOpt = 'keyMap: "' + keymap + '", cursorBlinkRate: 0, '
        kbLoad = '<script src="' + url_for('static', filename="codemirror/keymap/" + keymap + ".js", v=da_version) + '"></script>\n    '
    else:
        kbOpt = ''
        kbLoad = ''
    python_version = daconfig.get('python version', word('Unknown'))
    system_version = daconfig.get('system version', word('Unknown'))
    if python_version == system_version:
        version = word("Version ") + str(python_version)
    else:
        version = word("Version ") + str(python_version) + ' (Python); ' + str(system_version) + ' (' + word('system') + ')'
    response = make_response(render_template('pages/config.html', underlying_python_version=re.sub(r' \(.*', '', sys.version, flags=re.DOTALL), free_disk_space=humanize.naturalsize(disk_free), config_errors=docassemble.base.config.errors, config_messages=docassemble.base.config.env_messages, version_warning=version_warning, version=version, bodyclass='daadminbody', tab_title=word('Configuration'), page_title=word('Configuration'), extra_css=Markup('\n    <link href="' + url_for('static', filename='codemirror/lib/codemirror.css', v=da_version) + '" rel="stylesheet">\n    <link href="' + url_for('static', filename='codemirror/addon/search/matchesonscrollbar.css', v=da_version) + '" rel="stylesheet">\n    <link href="' + url_for('static', filename='codemirror/addon/display/fullscreen.css', v=da_version) + '" rel="stylesheet">\n    <link href="' + url_for('static', filename='codemirror/addon/scroll/simplescrollbars.css', v=da_version) + '" rel="stylesheet">\n    <link href="' + url_for('static', filename='app/pygments.min.css', v=da_version) + '" rel="stylesheet">'), extra_js=Markup('\n    <script src="' + url_for('static', filename="codemirror/lib/codemirror.js", v=da_version) + '"></script>\n    <script src="' + url_for('static', filename="codemirror/addon/search/searchcursor.js", v=da_version) + '"></script>\n    <script src="' + url_for('static', filename="codemirror/addon/scroll/annotatescrollbar.js", v=da_version) + '"></script>\n    <script src="' + url_for('static', filename="codemirror/addon/search/matchesonscrollbar.js", v=da_version) + '"></script>\n    <script src="' + url_for('static', filename="codemirror/addon/display/fullscreen.js", v=da_version) + '"></script>\n    <script src="' + url_for('static', filename="codemirror/addon/edit/matchbrackets.js", v=da_version) + '"></script>\n    <script src="' + url_for('static', filename="codemirror/mode/yaml/yaml.js", v=da_version) + '"></script>\n    ' + kbLoad + '<script>\n      daTextArea=document.getElementById("config_content");\n      daTextArea.value = JSON.parse(atob("' + safeid(json.dumps(content)) + '"));\n      var daCodeMirror = CodeMirror.fromTextArea(daTextArea, {mode: "yaml", ' + kbOpt + 'tabSize: 2, tabindex: 70, autofocus: true, lineNumbers: true, matchBrackets: true});\n      daCodeMirror.setOption("extraKeys", { Tab: function(cm) { var spaces = Array(cm.getOption("indentUnit") + 1).join(" "); cm.replaceSelection(spaces); }, "F11": function(cm) { cm.setOption("fullScreen", !cm.getOption("fullScreen")); }, "Esc": function(cm) { if (cm.getOption("fullScreen")) cm.setOption("fullScreen", false); }});\n      daCodeMirror.setOption("coverGutterNextToScrollbar", true);\n    </script>'), form=form), 200)
    response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
    return response

@app.route('/view_source', methods=['GET'])
@login_required
@roles_required(['developer', 'admin'])
def view_source():
    setup_translation()
    source_path = request.args.get('i', None)
    current_project = get_current_project()
    if source_path is None:
        logmessage("view_source: no source path")
        return ('File not found', 404)
    try:
        if re.search(r':', source_path):
            source = docassemble.base.parse.interview_source_from_string(source_path)
        else:
            try:
                source = docassemble.base.parse.interview_source_from_string('docassemble.playground' + str(current_user.id) + project_name(current_project) + ':' + source_path)
            except:
                source = docassemble.base.parse.interview_source_from_string(source_path)
    except Exception as errmess:
        logmessage("view_source: no source: " + str(errmess))
        return ('File not found', 404)
    header = source_path
    response = make_response(render_template('pages/view_source.html', version_warning=None, bodyclass='daadminbody', tab_title="Source", page_title="Source", extra_css=Markup('\n    <link href="' + url_for('static', filename='app/pygments.min.css') + '" rel="stylesheet">'), header=header, contents=Markup(highlight(source.content, YamlLexer(), HtmlFormatter(cssclass="highlight dafullheight")))), 200)
    response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
    return response

@app.route('/playgroundstatic/<current_project>/<userid>/<path:filename>', methods=['GET'])
def playground_static(current_project, userid, filename):
    if not app.config['ENABLE_PLAYGROUND']:
        return ('File not found', 404)
    #filename = re.sub(r'[^A-Za-z0-9\-\_\. ]', '', filename)
    try:
        attach = int(request.args.get('attach', 0))
    except:
        attach = 0
    area = SavedFile(userid, fix=True, section='playgroundstatic')
    the_directory = directory_for(area, current_project)
    filename = filename.replace('/', os.path.sep)
    path = os.path.join(the_directory, filename)
    if os.path.join('..', '') in path:
        return ('File not found', 404)
    if os.path.isfile(path):
        filename = os.path.basename(filename)
        extension, mimetype = get_ext_and_mimetype(filename)
        response = send_file(path, mimetype=str(mimetype))
        if attach:
            response.headers['Content-Disposition'] = 'attachment; filename=' + json.dumps(filename)
        return response
    return ('File not found', 404)

@app.route('/playgroundmodules/<current_project>/<userid>/<path:filename>', methods=['GET'])
@login_required
@roles_required(['developer', 'admin'])
def playground_modules(current_project, userid, filename):
    if not app.config['ENABLE_PLAYGROUND']:
        return ('File not found', 404)
    setup_translation()
    #filename = re.sub(r'[^A-Za-z0-9\-\_\. ]', '', filename)
    try:
        attach = int(request.args.get('attach', 0))
    except:
        attach = 0
    area = SavedFile(userid, fix=True, section='playgroundmodules')
    the_directory = directory_for(area, current_project)
    filename = filename.replace('/', os.path.sep)
    path = os.path.join(the_directory, filename)
    if os.path.join('..', '') in path:
        return ('File not found', 404)
    if os.path.isfile(path):
        filename = os.path.basename(filename)
        extension, mimetype = get_ext_and_mimetype(filename)
        response = send_file(path, mimetype=str(mimetype))
        if attach:
            response.headers['Content-Disposition'] = 'attachment; filename=' + json.dumps(filename)
        response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
        return response
    return ('File not found', 404)

@app.route('/playgroundsources/<current_project>/<userid>/<path:filename>', methods=['GET'])
@login_required
@roles_required(['developer', 'admin'])
def playground_sources(current_project, userid, filename):
    if not app.config['ENABLE_PLAYGROUND']:
        return ('File not found', 404)
    setup_translation()
    try:
        attach = int(request.args.get('attach', 0))
    except:
        attach = 0
    #filename = re.sub(r'[^A-Za-z0-9\-\_\(\)\. ]', '', filename)
    filename = filename.replace('/', os.path.sep)
    area = SavedFile(userid, fix=True, section='playgroundsources')
    reslt = write_ml_source(area, userid, current_project, filename)
    the_directory = directory_for(area, current_project)
    path = os.path.join(the_directory, filename)
    if os.path.join('..', '') in path:
        return ('File not found', 404)
    if os.path.isfile(path):
        filename = os.path.basename(filename)
        extension, mimetype = get_ext_and_mimetype(filename)
        response = send_file(path, mimetype=str(mimetype))
        if attach:
            response.headers['Content-Disposition'] = 'attachment; filename=' + json.dumps(filename)
        response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
        return response
    return ('File not found', 404)

@app.route('/playgroundtemplate/<current_project>/<userid>/<path:filename>', methods=['GET'])
@login_required
@roles_required(['developer', 'admin'])
def playground_template(current_project, userid, filename):
    if not app.config['ENABLE_PLAYGROUND']:
        return ('File not found', 404)
    #filename = re.sub(r'[^A-Za-z0-9\-\_\. ]', '', filename)
    setup_translation()
    try:
        attach = int(request.args.get('attach', 0))
    except:
        attach = 0
    area = SavedFile(userid, fix=True, section='playgroundtemplate')
    the_directory = directory_for(area, current_project)
    filename = filename.replace('/', os.path.sep)
    path = os.path.join(the_directory, filename)
    if os.path.join('..', '') in path:
        return ('File not found', 404)
    if os.path.isfile(path):
        filename = os.path.basename(filename)
        extension, mimetype = get_ext_and_mimetype(filename)
        response = send_file(path, mimetype=str(mimetype))
        if attach:
            response.headers['Content-Disposition'] = 'attachment; filename=' + json.dumps(filename)
        response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
        return response
    return ('File not found', 404)

@app.route('/playgrounddownload/<current_project>/<userid>/<path:filename>', methods=['GET'])
@login_required
@roles_required(['developer', 'admin'])
def playground_download(current_project, userid, filename):
    if not app.config['ENABLE_PLAYGROUND']:
        return ('File not found', 404)
    setup_translation()
    #filename = re.sub(r'[^A-Za-z0-9\-\_\. ]', '', filename)
    area = SavedFile(userid, fix=True, section='playground')
    the_directory = directory_for(area, current_project)
    filename = filename.replace('/', os.path.sep)
    path = os.path.join(the_directory, filename)
    if os.path.join('..', '') in path:
        return ('File not found', 404)
    if os.path.isfile(path):
        filename = os.path.basename(filename)
        extension, mimetype = get_ext_and_mimetype(path)
        response = send_file(path, mimetype=str(mimetype))
        response.headers['Content-type'] = 'text/plain; charset=utf-8'
        response.headers['Content-Disposition'] = 'attachment; filename=' + json.dumps(filename)
        response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
        return response
    return ('File not found', 404)

@app.route('/officefunctionfile', methods=['GET', 'POST'])
@cross_origin(origins='*', methods=['GET', 'POST', 'HEAD'], automatic_options=True)
def playground_office_functionfile():
    g.embed = True
    docassemble.base.functions.set_language(DEFAULT_LANGUAGE)
    if not app.config['ENABLE_PLAYGROUND']:
        return ('File not found', 404)
    functionform = FunctionFileForm(request.form)
    response = make_response(render_template('pages/officefunctionfile.html', current_project=get_current_project(), page_title=word("Docassemble Playground"), tab_title=word("Playground"), parent_origin=daconfig.get('office addin url', daconfig.get('url root', get_base_url())), form=functionform), 200)
    response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
    return response

@app.route('/officetaskpane', methods=['GET', 'POST'])
@cross_origin(origins='*', methods=['GET', 'POST', 'HEAD'], automatic_options=True)
def playground_office_taskpane():
    g.embed = True
    docassemble.base.functions.set_language(DEFAULT_LANGUAGE)
    if not app.config['ENABLE_PLAYGROUND']:
        return ('File not found', 404)
    defaultDaServer = url_for('rootindex', _external=True)
    response = make_response(render_template('pages/officeouter.html', page_title=word("Docassemble Playground"), tab_title=word("Playground"), defaultDaServer=defaultDaServer, extra_js=Markup("\n        <script>" + indent_by(variables_js(office_mode=True), 9) + "        </script>")), 200)
    response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
    return response

@app.route('/officeaddin', methods=['GET', 'POST'])
@cross_origin(origins='*', methods=['GET', 'POST', 'HEAD'], automatic_options=True)
@login_required
@roles_required(['developer', 'admin'])
def playground_office_addin():
    g.embed = True
    setup_translation()
    if not app.config['ENABLE_PLAYGROUND']:
        return ('File not found', 404)
    current_project = get_current_project()
    if request.args.get('fetchfiles', None):
        playground = SavedFile(current_user.id, fix=True, section='playground')
        the_directory = directory_for(playground, current_project)
        files = sorted([f for f in os.listdir(the_directory) if os.path.isfile(os.path.join(the_directory, f)) and re.search(r'^[A-Za-z0-9]', f)])
        return jsonify(success=True, files=files)
    pg_var_file = request.args.get('pgvars', None)
    #logmessage("playground_office_addin: YAML file is " + str(pg_var_file))
    use_html = request.args.get('html', False)
    uploadform = AddinUploadForm(request.form)
    if request.method == 'POST':
        area = SavedFile(current_user.id, fix=True, section='playgroundtemplate')
        filename = secure_filename(uploadform.filename.data)
        filename = re.sub(r'[^A-Za-z0-9\-\_\. ]+', '_', filename)
        if filename == '':
            return jsonify({'success': False})
        content = str(uploadform.content.data)
        start_index = 0
        char_index = 0
        for char in content:
            char_index += 1
            if char == ',':
                start_index = char_index
                break
        area.write_content(codecs.decode(bytearray(content[start_index:], encoding='utf-8'), 'base64'), filename=filename, binary=True)
        area.finalize()
        if use_html:
            if pg_var_file is None:
                pg_var_file = ''
        else:
            if pg_var_file is None or pg_var_file == '':
                return jsonify({'success': True, 'variables_json': [], 'vocab_list': []})
    if pg_var_file is not None:
        playground = SavedFile(current_user.id, fix=True, section='playground')
        the_directory = directory_for(playground, current_project)
        files = sorted([f for f in os.listdir(the_directory) if os.path.isfile(os.path.join(the_directory, f)) and re.search(r'^[A-Za-z0-9]', f)])
        if pg_var_file in files:
            #logmessage("playground_office_addin: file " + str(pg_var_file) + " was found")
            interview_source = docassemble.base.parse.interview_source_from_string('docassemble.playground' + str(current_user.id) + project_name(current_project) + ':' + pg_var_file)
            interview_source.set_testing(True)
        else:
            #logmessage("playground_office_addin: file " + str(pg_var_file) + " was not found")
            if pg_var_file == '':
                pg_var_file = 'test.yml'
            content = "modules:\n  - docassemble.base.util\n---\nmandatory: True\nquestion: hi"
            interview_source = docassemble.base.parse.InterviewSourceString(content=content, directory=the_directory, package="docassemble.playground" + str(current_user.id) + project_name(current_project), path="docassemble.playground" + str(current_user.id) + project_name(current_project) + ":" + pg_var_file, testing=True)
        interview = interview_source.get_interview()
        ensure_ml_file_exists(interview, pg_var_file, current_project)
        interview_status = docassemble.base.parse.InterviewStatus(current_info=current_info(yaml='docassemble.playground' + str(current_user.id) + project_name(current_project) + ':' + pg_var_file, req=request, action=None, device_id=request.cookies.get('ds', None)))
        if use_html:
            variables_html, vocab_list, vocab_dict = get_vars_in_use(interview, interview_status, debug_mode=False, show_messages=False, show_jinja_help=True, current_project=current_project)
            return jsonify({'success': True, 'current_project': current_project, 'variables_html': variables_html, 'vocab_list': list(vocab_list), 'vocab_dict': vocab_dict})
        else:
            variables_json, vocab_list = get_vars_in_use(interview, interview_status, debug_mode=False, return_json=True, current_project=current_project)
            return jsonify({'success': True, 'variables_json': variables_json, 'vocab_list': list(vocab_list)})
    parent_origin = re.sub(r'^(https?://[^/]+)/.*', r'\1', daconfig.get('office addin url', get_base_url()))
    response = make_response(render_template('pages/officeaddin.html', current_project=current_project, page_title=word("Docassemble Office Add-in"), tab_title=word("Office Add-in"), parent_origin=parent_origin, form=uploadform), 200)
    response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
    return response

def cloud_trash(use_gd, use_od, section, the_file, current_project):
    if use_gd:
        try:
            trash_gd_file(section, the_file, current_project)
        except Exception as the_err:
            logmessage("cloud_trash: unable to delete file on Google Drive.  " + str(the_err))
    elif use_od:
        try:
            trash_od_file(section, the_file, current_project)
        except Exception as the_err:
            try:
                logmessage("cloud_trash: unable to delete file on OneDrive.  " + str(the_err))
            except:
                logmessage("cloud_trash: unable to delete file on OneDrive.")

@app.route('/playgroundfiles', methods=['GET', 'POST'])
@login_required
@roles_required(['developer', 'admin'])
def playground_files():
    if not app.config['ENABLE_PLAYGROUND']:
        return ('File not found', 404)
    setup_translation()
    current_project = get_current_project()
    use_gd = bool(app.config['USE_GOOGLE_DRIVE'] is True and get_gd_folder() is not None)
    use_od = bool(use_gd is False and app.config['USE_ONEDRIVE'] is True and get_od_folder() is not None)
    form = PlaygroundFilesForm(request.form)
    formtwo = PlaygroundFilesEditForm(request.form)
    is_ajax = bool('ajax' in request.form and int(request.form['ajax']))
    section = werkzeug.utils.secure_filename(request.args.get('section', 'template'))
    the_file = secure_filename_spaces_ok(request.args.get('file', ''))
    scroll = False
    if the_file != '':
        scroll = True
    if request.method == 'GET':
        is_new = true_or_false(request.args.get('new', False))
    else:
        is_new = False
    if is_new:
        scroll = True
        the_file = ''
    if request.method == 'POST':
        form_validated = bool((form.purpose.data == 'upload' and form.validate()) or (formtwo.purpose.data == 'edit' and formtwo.validate()))
        if form_validated:
            if form.section.data:
                section = form.section.data
            if formtwo.file_name.data:
                the_file = formtwo.file_name.data
                the_file = re.sub(r'[^A-Za-z0-9\-\_\. ]+', '_', the_file)
    else:
        form_validated = None
    if section not in ("template", "static", "sources", "modules", "packages"):
        section = "template"
    pgarea = SavedFile(current_user.id, fix=True, section='playground')
    the_directory = directory_for(pgarea, current_project)
    if current_project != 'default' and not os.path.isdir(the_directory):
        current_project = set_current_project('default')
        the_directory = directory_for(pgarea, current_project)
    pulldown_files = sorted([f for f in os.listdir(the_directory) if os.path.isfile(os.path.join(the_directory, f)) and re.search(r'^[A-Za-z0-9]', f)])
    current_variable_file = get_variable_file(current_project)
    if current_variable_file is not None:
        if current_variable_file in pulldown_files:
            active_file = current_variable_file
        else:
            delete_variable_file(current_project)
            active_file = None
    else:
        active_file = None
    if active_file is None:
        current_file = get_current_file(current_project, 'questions')
        if current_file in pulldown_files:
            active_file = current_file
        elif len(pulldown_files) > 0:
            delete_current_file(current_project, 'questions')
            active_file = pulldown_files[0]
        else:
            delete_current_file(current_project, 'questions')
    area = SavedFile(current_user.id, fix=True, section='playground' + section)
    the_directory = directory_for(area, current_project)
    if request.args.get('delete', False):
        #argument = re.sub(r'[^A-Za-z0-9\-\_\. ]', '', request.args.get('delete'))
        argument = request.args.get('delete')
        if argument:
            the_directory = directory_for(area, current_project)
            the_file = add_project(argument, current_project)
            filename = os.path.join(the_directory, argument)
            if os.path.exists(filename):
                os.remove(filename)
                area.finalize()
                for key in r.keys('da:interviewsource:docassemble.playground' + str(current_user.id) + project_name(current_project) + ':*'):
                    r.incr(key.decode())
                cloud_trash(use_gd, use_od, section, argument, current_project)
                flash(word("Deleted file: ") + the_file, "success")
                for key in r.keys('da:interviewsource:docassemble.playground' + str(current_user.id) + project_name(current_project) + ':*'):
                    r.incr(key.decode())
                return redirect(url_for('playground_files', section=section, project=current_project))
            else:
                flash(word("File not found: ") + argument, "error")
    if request.args.get('convert', False):
        #argument = re.sub(r'[^A-Za-z0-9\-\_\. ]', '', request.args.get('convert'))
        argument = request.args.get('convert')
        if argument:
            filename = os.path.join(the_directory, argument)
            if os.path.exists(filename):
                to_file = os.path.splitext(argument)[0] + '.md'
                to_path = os.path.join(the_directory, to_file)
                if not os.path.exists(to_path):
                    extension, mimetype = get_ext_and_mimetype(argument)
                    if mimetype and mimetype in convertible_mimetypes:
                        the_format = convertible_mimetypes[mimetype]
                    elif extension and extension in convertible_extensions:
                        the_format = convertible_extensions[extension]
                    else:
                        flash(word("File format not understood: ") + argument, "error")
                        return redirect(url_for('playground_files', section=section, project=current_project))
                    result = word_to_markdown(filename, the_format)
                    if result is None:
                        flash(word("File could not be converted: ") + argument, "error")
                        return redirect(url_for('playground_files', section=section, project=current_project))
                    shutil.copyfile(result.name, to_path)
                    flash(word("Created new Markdown file called ") + to_file + word("."), "success")
                    area.finalize()
                    return redirect(url_for('playground_files', section=section, file=to_file, project=current_project))
            else:
                flash(word("File not found: ") + argument, "error")
    if request.method == 'POST' and form_validated:
        if 'uploadfile' in request.files:
            the_files = request.files.getlist('uploadfile')
            if the_files:
                need_to_restart = False
                for up_file in the_files:
                    try:
                        filename = werkzeug.utils.secure_filename(up_file.filename)
                        extension, mimetype = get_ext_and_mimetype(filename)
                        if section == 'modules' and extension != 'py':
                            flash(word("Sorry, only .py files can be uploaded here.  To upload other types of files, use other Folders."), 'error')
                            return redirect(url_for('playground_files', section=section, project=current_project))
                        filename = re.sub(r'[^A-Za-z0-9\-\_\. ]+', '_', filename)
                        the_file = filename
                        filename = os.path.join(the_directory, filename)
                        up_file.save(filename)
                        for key in r.keys('da:interviewsource:docassemble.playground' + str(current_user.id) + project_name(current_project) + ':*'):
                            r.incr(key.decode())
                        area.finalize()
                        if section == 'modules':
                            need_to_restart = True
                    except Exception as errMess:
                        flash("Error of type " + str(type(errMess)) + " processing upload: " + str(errMess), "error")
                if need_to_restart:
                    flash(word('Since you uploaded a Python module, the server needs to restart in order to load your module.'), 'info')
                    return redirect(url_for('restart_page', next=url_for('playground_files', section=section, file=the_file, project=current_project)))
                flash(word("Upload successful"), "success")
        if formtwo.delete.data:
            if the_file != '':
                filename = os.path.join(the_directory, the_file)
                if os.path.exists(filename):
                    os.remove(filename)
                    for key in r.keys('da:interviewsource:docassemble.playground' + str(current_user.id) + project_name(current_project) + ':*'):
                        r.incr(key.decode())
                    area.finalize()
                    flash(word("Deleted file: ") + the_file, "success")
                    return redirect(url_for('playground_files', section=section, project=current_project))
                else:
                    flash(word("File not found: ") + the_file, "error")
        if formtwo.submit.data and formtwo.file_content.data:
            if the_file != '':
                if section == 'modules' and not re.search(r'\.py$', the_file):
                    the_file = re.sub(r'\..*', '', the_file) + '.py'
                if formtwo.original_file_name.data and formtwo.original_file_name.data != the_file:
                    old_filename = os.path.join(the_directory, formtwo.original_file_name.data)
                    cloud_trash(use_gd, use_od, section, formtwo.original_file_name.data, current_project)
                    if os.path.isfile(old_filename):
                        os.remove(old_filename)
                filename = os.path.join(the_directory, the_file)
                with open(filename, 'w', encoding='utf-8') as fp:
                    fp.write(re.sub(r'\r\n', r'\n', formtwo.file_content.data))
                the_time = formatted_current_time()
                for key in r.keys('da:interviewsource:docassemble.playground' + str(current_user.id) + project_name(current_project) + ':*'):
                    r.incr(key.decode())
                area.finalize()
                if formtwo.active_file.data and formtwo.active_file.data != the_file:
                    #interview_file = os.path.join(pgarea.directory, formtwo.active_file.data)
                    r.incr('da:interviewsource:docassemble.playground' + str(current_user.id) + project_name(current_project) + ':' + formtwo.active_file.data)
                    #if os.path.isfile(interview_file):
                    #    with open(interview_file, 'a'):
                    #        os.utime(interview_file, None)
                    #    pgarea.finalize()
                flash_message = flash_as_html(str(the_file) + ' ' + word('was saved at') + ' ' + the_time + '.', message_type='success', is_ajax=is_ajax)
                if section == 'modules':
                    #restart_all()
                    flash(word('Since you changed a Python module, the server needs to restart in order to load your module.'), 'info')
                    return redirect(url_for('restart_page', next=url_for('playground_files', section=section, file=the_file, project=current_project)))
                if is_ajax:
                    return jsonify(success=True, flash_message=flash_message)
                else:
                    return redirect(url_for('playground_files', section=section, file=the_file, project=current_project))
            else:
                flash(word('You need to type in a name for the file'), 'error')
    if is_ajax and not form_validated:
        errors = []
        for fieldName, errorMessages in formtwo.errors.items():
            for err in errorMessages:
                errors.append(dict(fieldName=fieldName, err=err))
        return jsonify(success=False, errors=errors)
    files = sorted([f for f in os.listdir(the_directory) if os.path.isfile(os.path.join(the_directory, f)) and re.search(r'^[A-Za-z0-9]', f)])

    editable_files = []
    convertible_files = []
    trainable_files = {}
    mode = "yaml"
    for a_file in files:
        extension, mimetype = get_ext_and_mimetype(a_file)
        if (mimetype and mimetype in ok_mimetypes) or (extension and extension in ok_extensions) or (mimetype and mimetype.startswith('text')):
            if section == 'sources' and re.match(r'ml-.*\.json', a_file):
                trainable_files[a_file] = re.sub(r'^ml-|\.json$', '', a_file)
            else:
                editable_files.append(dict(name=a_file, modtime=os.path.getmtime(os.path.join(the_directory, a_file))))
    assign_opacity(editable_files)
    editable_file_listing = [x['name'] for x in editable_files]
    for a_file in files:
        extension, mimetype = get_ext_and_mimetype(a_file)
        b_file = os.path.splitext(a_file)[0] + '.md'
        if b_file not in editable_file_listing and ((mimetype and mimetype in convertible_mimetypes) or (extension and extension in convertible_extensions)):
            convertible_files.append(a_file)
    if the_file and not is_new and the_file not in editable_file_listing:
        the_file = ''
    if not the_file and not is_new:
        current_file = get_current_file(current_project, section)
        if current_file in editable_file_listing:
            the_file = current_file
        else:
            delete_current_file(current_project, section)
            if len(editable_files) > 0:
                the_file = sorted(editable_files, key=lambda x: x['modtime'])[-1]['name']
            else:
                if section == 'modules':
                    the_file = 'test.py'
                elif section == 'sources':
                    the_file = 'test.json'
                else:
                    the_file = 'test.md'
    if the_file in editable_file_listing:
        set_current_file(current_project, section, the_file)
    if the_file != '':
        extension, mimetype = get_ext_and_mimetype(the_file)
        if mimetype and mimetype in ok_mimetypes:
            mode = ok_mimetypes[mimetype]
        elif extension and extension in ok_extensions:
            mode = ok_extensions[extension]
        elif mimetype and mimetype.startswith('text'):
            mode = 'null'
    if mode != 'markdown':
        active_file = None
    if section == 'modules':
        mode = 'python'
    formtwo.original_file_name.data = the_file
    formtwo.file_name.data = the_file
    if the_file != '' and os.path.isfile(os.path.join(the_directory, the_file)):
        filename = os.path.join(the_directory, the_file)
    else:
        filename = None
    if filename is not None:
        area.finalize()
        with open(filename, 'r', encoding='utf-8') as fp:
            try:
                content = fp.read()
            except:
                filename = None
                content = ''
    elif formtwo.file_content.data:
        content = re.sub(r'\r\n', r'\n', formtwo.file_content.data)
    else:
        content = ''
    lowerdescription = None
    description = None
    if section == "template":
        header = word("Templates")
        description = 'Add files here that you want want to include in your interviews using <a target="_blank" href="https://docassemble.org/docs/documents.html#docx template file"><code>docx template file</code></a>, <a target="_blank" href="https://docassemble.org/docs/documents.html#pdf template file"><code>pdf template file</code></a>, <a target="_blank" href="https://docassemble.org/docs/documents.html#content file"><code>content file</code></a>, <a target="_blank" href="https://docassemble.org/docs/documents.html#initial yaml"><code>initial yaml</code></a>, <a target="_blank" href="https://docassemble.org/docs/documents.html#additional yaml"><code>additional yaml</code></a>, <a target="_blank" href="https://docassemble.org/docs/documents.html#template file"><code>template file</code></a>, <a target="_blank" href="https://docassemble.org/docs/documents.html#rtf template file"><code>rtf template file</code></a>, or <a target="_blank" href="https://docassemble.org/docs/documents.html#docx reference file"><code>docx reference file</code></a>.'
        upload_header = word("Upload a template file")
        list_header = word("Existing template files")
        edit_header = word('Edit text files')
        after_text = None
    elif section == "static":
        header = word("Static Files")
        description = 'Add files here that you want to include in your interviews with <a target="_blank" href="https://docassemble.org/docs/initial.html#images"><code>images</code></a>, <a target="_blank" href="https://docassemble.org/docs/initial.html#image sets"><code>image sets</code></a>, <a target="_blank" href="https://docassemble.org/docs/markup.html#inserting%20images"><code>[FILE]</code></a> or <a target="_blank" href="https://docassemble.org/docs/functions.html#url_of"><code>url_of()</code></a>.'
        upload_header = word("Upload a static file")
        list_header = word("Existing static files")
        edit_header = word('Edit text files')
        after_text = None
    elif section == "sources":
        header = word("Source Files")
        description = 'Add files here that you want to use as a data source in your interview code, such as word translation files and training data for machine learning.  For Python source code, see the Modules folder.'
        upload_header = word("Upload a source file")
        list_header = word("Existing source files")
        edit_header = word('Edit source files')
        after_text = None
    elif section == "modules":
        header = word("Modules")
        upload_header = word("Upload a Python module")
        list_header = word("Existing module files")
        edit_header = word('Edit module files')
        description = 'You can use this page to add Python module files (.py files) that you want to include in your interviews using <a target="_blank" href="https://docassemble.org/docs/initial.html#modules"><code>modules</code></a> or <a target="_blank" href="https://docassemble.org/docs/initial.html#imports"><code>imports</code></a>.'
        lowerdescription = Markup("""<p>To use this in an interview, write a <a target="_blank" href="https://docassemble.org/docs/initial.html#modules"><code>modules</code></a> block that refers to this module using Python's syntax for specifying a "relative import" of a module (i.e., prefix the module name with a period).</p>""" + highlight('---\nmodules:\n  - .' + re.sub(r'\.py$', '', the_file) + '\n---', YamlLexer(), HtmlFormatter()))
        after_text = None
    if scroll:
        extra_command = """
        if ($("#file_name").val().length > 0){
          daCodeMirror.focus();
        }
        else{
          $("#file_name").focus()
        }
        scrollBottom();"""
    else:
        extra_command = ""
    if keymap:
        kbOpt = 'keyMap: "' + keymap + '", cursorBlinkRate: 0, '
        kbLoad = '<script src="' + url_for('static', filename="codemirror/keymap/" + keymap + ".js", v=da_version) + '"></script>\n    '
    else:
        kbOpt = ''
        kbLoad = ''
    extra_js = """
    <script>
      var daNotificationContainer = """ + json.dumps(NOTIFICATION_CONTAINER) + """;
      var daNotificationMessage = """ + json.dumps(NOTIFICATION_MESSAGE) + """;
      Object.defineProperty(String.prototype, "daSprintf", {
        value: function () {
          var args = Array.from(arguments),
            i = 0;
          function defaultNumber(iValue) {
            return iValue != undefined && !isNaN(iValue) ? iValue : "0";
          }
          function defaultString(iValue) {
            return iValue == undefined ? "" : "" + iValue;
          }
          return this.replace(
            /%%|%([+\\-])?([^1-9])?(\\d+)?(\\.\\d+)?([deEfhHioQqs])/g,
            function (match, sign, filler, scale, precision, type) {
              var strOut, space, value;
              var asNumber = false;
              if (match == "%%") return "%";
              if (i >= args.length) return match;
              value = args[i];
              while (Array.isArray(value)) {
                args.splice(i, 1);
                for (var j = i; value.length > 0; j++)
                  args.splice(j, 0, value.shift());
                value = args[i];
              }
              i++;
              if (filler == undefined) filler = " "; // default
              if (scale == undefined && !isNaN(filler)) {
                scale = filler;
                filler = " ";
              }
              if (sign == undefined) sign = "sqQ".indexOf(type) >= 0 ? "+" : "-"; // default
              if (scale == undefined) scale = 0; // default
              if (precision == undefined) precision = ".0"; // default
              scale = parseInt(scale);
              precision = parseInt(precision.substr(1));
              switch (type) {
                case "d":
                case "i":
                  // decimal integer
                  asNumber = true;
                  strOut = parseInt(defaultNumber(value));
                  if (precision > 0) strOut += "." + "0".repeat(precision);
                  break;
                case "e":
                case "E":
                  // float in exponential notation
                  asNumber = true;
                  strOut = parseFloat(defaultNumber(value));
                  if (precision == 0) strOut = strOut.toExponential();
                  else strOut = strOut.toExponential(precision);
                  if (type == "E") strOut = strOut.replace("e", "E");
                  break;
                case "f":
                  // decimal float
                  asNumber = true;
                  strOut = parseFloat(defaultNumber(value));
                  if (precision != 0) strOut = strOut.toFixed(precision);
                  break;
                case "o":
                case "h":
                case "H":
                  // Octal or Hexagesimal integer notation
                  strOut =
                    "\\\\" +
                    (type == "o" ? "0" : type) +
                    parseInt(defaultNumber(value)).toString(type == "o" ? 8 : 16);
                  break;
                case "q":
                  // single quoted string
                  strOut = "'" + defaultString(value) + "'";
                  break;
                case "Q":
                  // double quoted string
                  strOut = '"' + defaultString(value) + '"';
                  break;
                default:
                  // string
                  strOut = defaultString(value);
                  break;
              }
              if (typeof strOut != "string") strOut = "" + strOut;
              if ((space = strOut.length) < scale) {
                if (asNumber) {
                  if (sign == "-") {
                    if (strOut.indexOf("-") < 0)
                      strOut = filler.repeat(scale - space) + strOut;
                    else
                      strOut =
                        "-" +
                        filler.repeat(scale - space) +
                        strOut.replace("-", "");
                  } else {
                    if (strOut.indexOf("-") < 0)
                      strOut = "+" + filler.repeat(scale - space - 1) + strOut;
                    else
                      strOut =
                        "-" +
                        filler.repeat(scale - space) +
                        strOut.replace("-", "");
                  }
                } else {
                  if (sign == "-") strOut = filler.repeat(scale - space) + strOut;
                  else strOut = strOut + filler.repeat(scale - space);
                }
              } else if (asNumber && sign == "+" && strOut.indexOf("-") < 0)
                strOut = "+" + strOut;
              return strOut;
            }
          );
        },
      });
      Object.defineProperty(window, "daSprintf", {
        value: function (str, ...rest) {
          if (typeof str == "string")
            return String.prototype.daSprintf.apply(str, rest);
          return "";
        },
      });
      var daCodeMirror;
      var daTextArea;
      var vocab = [];
      var currentFile = """ + json.dumps(the_file) + """;
      var daIsNew = """ + ('true' if is_new else 'false') + """;
      var existingFiles = """ + json.dumps(files) + """;
      var daSection = """ + '"' + section + '";' + """
      var attrs_showing = Object();
      var currentProject = """ + json.dumps(current_project) + """;
""" + indent_by(variables_js(form='formtwo'), 6) + """
""" + indent_by(search_js(form='formtwo'), 6) + """
      var daExpireSession = null;
      function resetExpireSession(){
        if (daExpireSession != null){
          window.clearTimeout(daExpireSession);
        }
        daExpireSession = setTimeout(function(){
          alert(""" + json.dumps(word("Your browser session has expired and you have been signed out.  You will not be able to save your work.  Please log in again.")) + """);
        }, """ + str(999 * int(daconfig.get('session lifetime seconds', 43200))) + """);
      }
      function saveCallback(data){
        if (!data.success){
          var n = data.errors.length;
          for (var i = 0; i < n; ++i){
            $('input[name="' + data.errors[i].fieldName + '"]').parents('.input-group').addClass("da-group-has-error").after('<div class="da-has-error invalid-feedback">' + data.errors[i].err + '</div>');
          }
          return;
        }
        $('.da-has-error').remove();
        $('.da-group-has-error').removeClass('da-group-has-error');
        fetchVars(true);
        if ($("#daflash").length){
          $("#daflash").html(data.flash_message);
        }
        else{
          $("#damain").prepend(daSprintf(daNotificationContainer, data.flash_message));
        }
      }
      function scrollBottom(){
        $("html, body").animate({
          scrollTop: $("#editnav").offset().top - 53
        }, "slow");
      }
      $( document ).ready(function() {
        resetExpireSession();
        $("#file_name").on('change', function(){
          var newFileName = $(this).val();
          if ((!daIsNew) && newFileName == currentFile){
            return;
          }
          for (var i = 0; i < existingFiles.length; i++){
            if (newFileName == existingFiles[i]){
              alert(""" + json.dumps(word("Warning: a file by that name already exists.  If you save, you will overwrite it.")) + """);
              return;
            }
          }
          return;
        });
        $("#dauploadbutton").click(function(event){
          if ($("#uploadfile").val() == ""){
            event.preventDefault();
            return false;
          }
        });
        daTextArea = document.getElementById("file_content");
        daCodeMirror = CodeMirror.fromTextArea(daTextArea, {mode: """ + ('{name: "markdown", underscoresBreakWords: false}' if mode == 'markdown' else json.dumps(mode)) + """, """ + kbOpt + """tabSize: 2, tabindex: 580, autofocus: false, lineNumbers: true, matchBrackets: true, lineWrapping: """ + ('true' if daconfig.get('wrap lines in playground', True) else 'false') + """});
        $(window).bind("beforeunload", function(){
          daCodeMirror.save();
          $("#formtwo").trigger("checkform.areYouSure");
        });
        $("#daDelete").click(function(event){
          if (!confirm(""" + json.dumps(word("Are you sure that you want to delete this file?")) + """)){
            event.preventDefault();
          }
        });
        $("#formtwo").areYouSure(""" + json.dumps(json.dumps({'message': word("There are unsaved changes.  Are you sure you wish to leave this page?")})) + """);
        $("#formtwo").bind("submit", function(e){
          daCodeMirror.save();
          $("#formtwo").trigger("reinitialize.areYouSure");
          if (daSection != 'modules' && !daIsNew){
            var extraVariable = ''
            if ($("#daVariables").length){
              extraVariable = '&active_file=' + encodeURIComponent($("#daVariables").val());
            }
            $.ajax({
              type: "POST",
              url: """ + '"' + url_for('playground_files', project=current_project) + '"' + """,
              data: $("#formtwo").serialize() + extraVariable + '&submit=Save&ajax=1',
              success: function(data){
                if (data.action && data.action == 'reload'){
                  location.reload(true);
                }
                resetExpireSession();
                saveCallback(data);
                setTimeout(function(){
                  $("#daflash .alert-success").hide(300, function(){
                    $(self).remove();
                  });
                }, 3000);
              },
              dataType: 'json'
            });
            e.preventDefault();
            return false;
          }
          return true;
        });
        daCodeMirror.setOption("extraKeys", { Tab: function(cm) { var spaces = Array(cm.getOption("indentUnit") + 1).join(" "); cm.replaceSelection(spaces); }, "F11": function(cm) { cm.setOption("fullScreen", !cm.getOption("fullScreen")); }, "Esc": function(cm) { if (cm.getOption("fullScreen")) cm.setOption("fullScreen", false); }});
        daCodeMirror.setOption("coverGutterNextToScrollbar", true);
        searchReady();
        variablesReady();
        fetchVars(false);""" + extra_command + """
      });
      searchReady();
      $('#uploadfile').on('change', function(){
        var fileName = $(this).val();
        fileName = fileName.replace(/.*\\\\/, '');
        fileName = fileName.replace(/.*\\//, '');
        $(this).next('.custom-file-label').html(fileName);
      });
    </script>"""
    if keymap:
        kbOpt = 'keyMap: "' + keymap + '", cursorBlinkRate: 0, '
        kbLoad = '<script src="' + url_for('static', filename="codemirror/keymap/" + keymap + ".js") + '"></script>\n    '
    else:
        kbOpt = ''
        kbLoad = ''
    any_files = bool(len(editable_files) > 0)
    back_button = Markup('<span class="navbar-brand navbar-nav dabackicon me-3"><a href="' + url_for('playground_page', project=current_project) + '" class="dabackbuttoncolor nav-link" title=' + json.dumps(word("Go back to the main Playground page")) + '><i class="fas fa-chevron-left"></i><span class="daback">' + word('Back') + '</span></a></span>')
    cm_mode = ''
    if mode == 'null':
        modes = []
    elif mode == 'htmlmixed':
        modes = ['css', 'xml', 'htmlmixed']
    else:
        modes = [mode]
    for the_mode in modes:
        cm_mode += '\n    <script src="' + url_for('static', filename="codemirror/mode/" + the_mode + "/" + ('damarkdown' if the_mode == 'markdown' else the_mode) + ".js", v=da_version) + '"></script>'
    if current_project != 'default':
        header += " / " + current_project
    response = make_response(render_template('pages/playgroundfiles.html', current_project=current_project, version_warning=None, bodyclass='daadminbody', use_gd=use_gd, use_od=use_od, back_button=back_button, tab_title=header, page_title=header, extra_css=Markup('\n    <link href="' + url_for('static', filename='app/playgroundbundle.css', v=da_version) + '" rel="stylesheet">'), extra_js=Markup('\n    <script src="' + url_for('static', filename="app/playgroundbundle.js", v=da_version) + '"></script>\n    ' + kbLoad + cm_mode + extra_js), header=header, upload_header=upload_header, list_header=list_header, edit_header=edit_header, description=Markup(description), lowerdescription=lowerdescription, form=form, files=sorted(files, key=lambda y: y.lower()), section=section, userid=current_user.id, editable_files=sorted(editable_files, key=lambda y: y['name'].lower()), editable_file_listing=editable_file_listing, trainable_files=trainable_files, convertible_files=convertible_files, formtwo=formtwo, current_file=the_file, content=content, after_text=after_text, is_new=str(is_new), any_files=any_files, pulldown_files=sorted(pulldown_files, key=lambda y: y.lower()), active_file=active_file, playground_package='docassemble.playground' + str(current_user.id) + project_name(current_project)), 200)
    response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
    return response

@app.route('/pullplaygroundpackage', methods=['GET', 'POST'])
@login_required
@roles_required(['developer', 'admin'])
def pull_playground_package():
    setup_translation()
    if not app.config['ENABLE_PLAYGROUND']:
        return ('File not found', 404)
    current_project = get_current_project()
    form = PullPlaygroundPackage(request.form)
    if request.method == 'POST':
        if form.pull.data:
            if form.github_url.data and form.pypi.data:
                flash(word("You cannot pull from GitHub and PyPI at the same time.  Please fill in one and leave the other blank."), 'error')
            elif form.github_url.data:
                return redirect(url_for('playground_packages', project=current_project, pull='1', github_url=re.sub(r'/*$', '', str(form.github_url.data).strip()), branch=form.github_branch.data))
            elif form.pypi.data:
                return redirect(url_for('playground_packages', project=current_project, pull='1', pypi=form.pypi.data))
        if form.cancel.data:
            return redirect(url_for('playground_packages', project=current_project))
    elif 'github' in request.args:
        form.github_url.data = re.sub(r'[^A-Za-z0-9\-\.\_\~\:\/\?\#\[\]\@\!\$\&\'\(\)\*\+\,\;\=\`]', '', request.args['github'])
    elif 'pypi' in request.args:
        form.pypi.data = re.sub(r'[^A-Za-z0-9\-\.\_\~\:\/\?\#\[\]\@\!\$\&\'\(\)\*\+\,\;\=\`]', '', request.args['pypi'])
    form.github_branch.choices = []
    description = word("Enter a URL of a GitHub repository containing an extension package.  When you press Pull, the contents of that repository will be copied into the Playground, overwriting any files with the same names.  Or, put in the name of a PyPI package and it will do the same with the package on PyPI.")
    branch = request.args.get('branch')
    extra_js = """
    <script>
      var default_branch = """ + json.dumps(branch if branch else GITHUB_BRANCH) + """;
      function get_branches(){
        var elem = $("#github_branch");
        elem.empty();
        var opt = $("<option><\/option>");
        opt.attr("value", "").text("Not applicable");
        elem.append(opt);
        var github_url = $("#github_url").val();
        if (!github_url){
          return;
        }
        $.get(""" + json.dumps(url_for('get_git_branches')) + """, { url: github_url }, "json")
        .done(function(data){
          //console.log(data);
          if (data.success){
            var n = data.result.length;
            if (n > 0){
              var default_to_use = default_branch;
              var to_try = [default_branch, """ + json.dumps(GITHUB_BRANCH) + """, 'master', 'main'];
            outer:
              for (var j = 0; j < 4; j++){
                for (var i = 0; i < n; i++){
                  if (data.result[i].name == to_try[j]){
                    default_to_use = to_try[j];
                    break outer;
                  }
                }
              }
              elem.empty();
              for (var i = 0; i < n; i++){
                opt = $("<option><\/option>");
                opt.attr("value", data.result[i].name).text(data.result[i].name);
                if (data.result[i].name == default_to_use){
                  opt.prop('selected', true);
                }
                $(elem).append(opt);
              }
            }
          }
        });
      }
      $( document ).ready(function() {
        get_branches();
        $("#github_url").on('change', get_branches);
      });
    </script>
"""
    response = make_response(render_template('pages/pull_playground_package.html',
                                             current_project=current_project,
                                             form=form,
                                             description=description,
                                             version_warning=version_warning,
                                             bodyclass='daadminbody',
                                             title=word("Pull GitHub or PyPI Package"),
                                             tab_title=word("Pull"),
                                             page_title=word("Pull"),
                                             extra_js=Markup(extra_js)), 200)
    response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
    return response

def get_branches_of_repo(giturl):
    repo_name = re.sub(r'/*$', '', giturl)
    m = re.search(r'//(.+):x-oauth-basic@github.com', repo_name)
    if m:
        access_token = m.group(1)
    else:
        access_token = None
    repo_name = re.sub(r'^http.*github.com/', '', repo_name)
    repo_name = re.sub(r'.*@github.com:', '', repo_name)
    repo_name = re.sub(r'.git$', '', repo_name)
    if app.config['USE_GITHUB']:
        github_auth = r.get('da:using_github:userid:' + str(current_user.id))
    else:
        github_auth = None
    if github_auth and access_token is None:
        storage = RedisCredStorage(app='github')
        credentials = storage.get()
        if not credentials or credentials.invalid:
            http = httplib2.Http()
        else:
            http = credentials.authorize(httplib2.Http())
    else:
        http = httplib2.Http()
    the_url = "https://api.github.com/repos/" + repo_name + '/branches'
    branches = []
    if access_token:
        resp, content = http.request(the_url, "GET", headers=dict(Authorization="token " + access_token))
    else:
        resp, content = http.request(the_url, "GET")
    if int(resp['status']) == 200:
        branches.extend(json.loads(content.decode()))
        while True:
            next_link = get_next_link(resp)
            if next_link:
                if access_token:
                    resp, content = http.request(next_link, "GET", headers=dict(Authorization="token " + access_token))
                else:
                    resp, content = http.request(next_link, "GET")
                if int(resp['status']) != 200:
                    raise Exception(repo_name + " fetch failed")
                else:
                    branches.extend(json.loads(content.decode()))
            else:
                break
        return branches
    raise Exception(the_url + " fetch failed on first try; got " + str(resp['status']))

def get_repo_info(giturl):
    repo_name = re.sub(r'/*$', '', giturl)
    m = re.search(r'//(.+):x-oauth-basic@github.com', repo_name)
    if m:
        access_token = m.group(1)
    else:
        access_token = None
    repo_name = re.sub(r'^http.*github.com/', '', repo_name)
    repo_name = re.sub(r'.*@github.com:', '', repo_name)
    repo_name = re.sub(r'.git$', '', repo_name)
    if app.config['USE_GITHUB']:
        github_auth = r.get('da:using_github:userid:' + str(current_user.id))
    else:
        github_auth = None
    if github_auth and access_token is None:
        storage = RedisCredStorage(app='github')
        credentials = storage.get()
        if not credentials or credentials.invalid:
            http = httplib2.Http()
        else:
            http = credentials.authorize(httplib2.Http())
    else:
        http = httplib2.Http()
    the_url = "https://api.github.com/repos/" + repo_name
    if access_token:
        resp, content = http.request(the_url, "GET", headers=dict(Authorization="token " + access_token))
    else:
        resp, content = http.request(the_url, "GET")
    if int(resp['status']) == 200:
        return json.loads(content.decode())
    raise Exception(the_url + " fetch failed on first try; got " + str(resp['status']))

@app.route('/get_git_branches', methods=['GET'])
@login_required
@roles_required(['developer', 'admin'])
def get_git_branches():
    if not app.config['ENABLE_PLAYGROUND']:
        return ('File not found', 404)
    if 'url' not in request.args:
        return ('File not found', 404)
    giturl = request.args['url'].strip()
    try:
        return jsonify(dict(success=True, result=get_branches_of_repo(giturl)))
    except Exception as err:
        return jsonify(dict(success=False, reason=str(err)))

def get_user_repositories(http):
    repositories = []
    resp, content = http.request("https://api.github.com/user/repos", "GET")
    if int(resp['status']) == 200:
        repositories.extend(json.loads(content.decode()))
        while True:
            next_link = get_next_link(resp)
            if next_link:
                resp, content = http.request(next_link, "GET")
                if int(resp['status']) != 200:
                    raise DAError("get_user_repositories: could not get information from next URL")
                else:
                    repositories.extend(json.loads(content.decode()))
            else:
                break
    else:
        raise DAError("playground_packages: could not get information about repositories")
    return repositories

def get_orgs_info(http):
    orgs_info = []
    resp, content = http.request("https://api.github.com/user/orgs", "GET")
    if int(resp['status']) == 200:
        orgs_info.extend(json.loads(content.decode()))
        while True:
            next_link = get_next_link(resp)
            if next_link:
                resp, content = http.request(next_link, "GET")
                if int(resp['status']) != 200:
                    raise DAError("get_orgs_info: could not get additional information about organizations")
                else:
                    orgs_info.extend(json.loads(content.decode()))
            else:
                break
    else:
        raise DAError("get_orgs_info: failed to get orgs using https://api.github.com/user/orgs")
    return orgs_info

def get_branch_info(http, full_name):
    branch_info = []
    resp, content = http.request("https://api.github.com/repos/" + str(full_name) + '/branches', "GET")
    if int(resp['status']) == 200:
        branch_info.extend(json.loads(content.decode()))
        while True:
            next_link = get_next_link(resp)
            if next_link:
                resp, content = http.request(next_link, "GET")
                if int(resp['status']) != 200:
                    raise DAError("get_branch_info: could not get additional information from next URL")
                else:
                    branch_info.extend(json.loads(content))
            else:
                break
    else:
        logmessage("get_branch_info: could not get info from https://api.github.com/repos/" + str(full_name) + '/branches')
    return branch_info

def fix_package_folder():
    use_gd = bool(app.config['USE_GOOGLE_DRIVE'] is True and get_gd_folder() is not None)
    use_od = bool(use_gd is False and app.config['USE_ONEDRIVE'] is True and get_od_folder() is not None)
    problem_exists = False
    area = SavedFile(current_user.id, fix=True, section='playgroundpackages')
    for f in os.listdir(area.directory):
        path = os.path.join(area.directory, f)
        if os.path.isfile(path) and not f.startswith('docassemble.') and not f.startswith('.'):
            os.rename(path, os.path.join(area.directory, 'docassemble.' + f))
            cloud_trash(use_gd, use_od, 'packages', f, 'default')
            problem_exists = True
        if os.path.isdir(path) and not f.startswith('.'):
            for e in os.listdir(path):
                if os.path.isfile(os.path.join(path, e)) and not e.startswith('docassemble.') and not e.startswith('.'):
                    os.rename(os.path.join(path, e), os.path.join(path, 'docassemble.' + e))
                    cloud_trash(use_gd, use_od, 'packages', e, f)
                    problem_exists = True
    if problem_exists:
        area.finalize()

def do_playground_pull(area, current_project, github_url=None, branch=None, pypi_package=None, can_publish_to_github=False, github_user_name=None, github_email=None, pull_only=False):
    area_sec = dict(templates='playgroundtemplate', static='playgroundstatic', sources='playgroundsources', questions='playground')
    readme_text = ''
    setup_py = ''
    if branch in ('', 'None'):
        branch = None
    if branch:
        branch = werkzeug.utils.secure_filename(branch)
        branch_option = '-b "' + branch + '" '
    else:
        branch_option = ''
    need_to_restart = False
    extracted = {}
    data_files = dict(templates=[], static=[], sources=[], interviews=[], modules=[], questions=[])
    directory = tempfile.mkdtemp()
    output = ''
    pypi_url = daconfig.get('pypi url', 'https://pypi.python.org/pypi')
    expected_name = 'unknown'
    if github_url:
        github_url = re.sub(r'[^A-Za-z0-9\-\.\_\~\:\/\#\[\]\@\$\+\,\=]', '', github_url)
        repo_name = re.sub(r'/*$', '', github_url)
        repo_name = re.sub(r'^http.*github.com/', '', repo_name)
        repo_name = re.sub(r'.*@github.com:', '', repo_name)
        repo_name = re.sub(r'.git$', '', repo_name)
        if not 'x-oauth-basic@github.com' in github_url and can_publish_to_github and github_email:
            github_url = f'git@github.com:{repo_name}.git'
            expected_name = re.sub(r'.*/', '', github_url)
            expected_name = re.sub(r'\.git', '', expected_name)
            expected_name = re.sub(r'docassemble-', '', expected_name)
            (private_key_file, public_key_file) = get_ssh_keys(github_email)
            os.chmod(private_key_file, stat.S_IRUSR | stat.S_IWUSR)
            os.chmod(public_key_file, stat.S_IRUSR | stat.S_IWUSR)
            ssh_script = tempfile.NamedTemporaryFile(mode='w', prefix="datemp", suffix='.sh', delete=False, encoding='utf-8')
            ssh_script.write('# /bin/bash\n\nssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o GlobalKnownHostsFile=/dev/null -i "' + str(private_key_file) + '" $1 $2 $3 $4 $5 $6')
            ssh_script.close()
            os.chmod(ssh_script.name, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR )
            #git_prefix = "GIT_SSH_COMMAND='ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o GlobalKnownHostsFile=/dev/null -i \"" + str(private_key_file) + "\"' "
            git_prefix = "GIT_SSH=" + ssh_script.name + " "
            output += "Doing " + git_prefix + "git clone " + branch_option + github_url + "\n"
            try:
                output += subprocess.check_output(git_prefix + "git clone " + branch_option + '"' + github_url + '"', cwd=directory, stderr=subprocess.STDOUT, shell=True).decode()
            except subprocess.CalledProcessError as err:
                output += err.output.decode()
                return dict(action="error", message="error running git clone.  " + output)
        else:
            if not github_url.startswith('http'):
                github_url = f'https://github.com/{repo_name}'
            expected_name = re.sub(r'.*/', '', github_url)
            expected_name = re.sub(r'\.git', '', expected_name)
            expected_name = re.sub(r'docassemble-', '', expected_name)
            try:
                if branch is not None:
                    logmessage("Doing git clone -b " + branch + " " + github_url)
                    output += subprocess.check_output(['git', 'clone', '-b', branch, github_url], cwd=directory, stderr=subprocess.STDOUT).decode()
                else:
                    logmessage("Doing git clone " + github_url)
                    output += subprocess.check_output(['git', 'clone', github_url], cwd=directory, stderr=subprocess.STDOUT).decode()
            except subprocess.CalledProcessError as err:
                output += err.output.decode()
                return dict(action="error", message="error running git clone.  " + output)
        logmessage(output)
        dirs_inside = [f for f in os.listdir(directory) if os.path.isdir(os.path.join(directory, f)) and re.search(r'^[A-Za-z0-9]', f)]
        if len(dirs_inside) == 1:
            commit_file = os.path.join(directory_for(area['playgroundpackages'], current_project), '.' + dirs_inside[0])
            packagedir = os.path.join(directory, dirs_inside[0])
            try:
                current_commit = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=packagedir, stderr=subprocess.STDOUT).decode()
            except subprocess.CalledProcessError as err:
                output = err.output.decode()
                return dict(action="error", message="error running git rev-parse.  " + output)
            with open(commit_file, 'w', encoding='utf-8') as fp:
                fp.write(current_commit.strip())
            logmessage("Wrote " + current_commit.strip() + " to " + commit_file)
        else:
            logmessage("Did not find a single directory inside repo")
        if pull_only:
            return dict(action='pull_only')
    elif pypi_package:
        pypi_package = re.sub(r'[^A-Za-z0-9\-\.\_\:\/\@\+\=]', '', pypi_package)
        pypi_package = 'docassemble.' + re.sub(r'^docassemble\.', '', pypi_package)
        package_file = tempfile.NamedTemporaryFile(suffix='.tar.gz')
        try:
            http = httplib2.Http()
            resp, content = http.request("https://pypi.python.org/pypi/" + str(pypi_package) + "/json", "GET")
            pypi_url = None
            if int(resp['status']) == 200:
                pypi_response = json.loads(content.decode())
                for file_option in pypi_response['releases'][pypi_response['info']['version']]:
                    if file_option['packagetype'] == 'sdist':
                        pypi_url = file_option['url']
                        break
            else:
                return dict(action='fail', message=word("The package you specified could not be downloaded from PyPI."))
            if pypi_url is None:
                return dict(action='fail', message=word("The package you specified could not be downloaded from PyPI as a tar.gz file."))
        except Exception as err:
            return dict(action='error', message="error getting information about PyPI package.  " + str(err))
        try:
            urlretrieve(pypi_url, package_file.name)
        except Exception as err:
            return dict(action='error', message="error downloading PyPI package.  " + str(err))
        try:
            tar = tarfile.open(package_file.name)
            tar.extractall(path=directory)
            tar.close()
        except Exception as err:
            return dict(action='error', message="error unpacking PyPI package.  " + str(err))
        package_file.close()
    initial_directories = len(splitall(directory)) + 1
    for root, dirs, files in os.walk(directory):
        at_top_level = bool('setup.py' in files and 'docassemble' in dirs)
        for a_file in files:
            orig_file = os.path.join(root, a_file)
            #output += "Original file is " + orig_file + "\n"
            thefilename = os.path.join(*splitall(orig_file)[initial_directories:])
            (the_directory, filename) = os.path.split(thefilename)
            if filename.startswith('#') or filename.endswith('~'):
                continue
            dirparts = splitall(the_directory)
            if '.git' in dirparts:
                continue
            levels = re.findall(r'/', the_directory)
            for sec in ('templates', 'static', 'sources', 'questions'):
                if the_directory.endswith('data/' + sec) and filename != 'README.md':
                    data_files[sec].append(filename)
                    target_filename = os.path.join(directory_for(area[area_sec[sec]], current_project), filename)
                    #output += "Copying " + orig_file + "\n"
                    copy_if_different(orig_file, target_filename)
            if filename == 'README.md' and at_top_level:
                with open(orig_file, 'r', encoding='utf-8') as fp:
                    readme_text = fp.read()
            if filename == 'setup.py' and at_top_level:
                with open(orig_file, 'r', encoding='utf-8') as fp:
                    setup_py = fp.read()
            elif len(levels) >= 1 and filename.endswith('.py') and filename != '__init__.py' and 'tests' not in dirparts and 'data' not in dirparts:
                data_files['modules'].append(filename)
                target_filename = os.path.join(directory_for(area['playgroundmodules'], current_project), filename)
                #output += "Copying " + orig_file + "\n"
                if (not os.path.isfile(target_filename)) or filecmp.cmp(orig_file, target_filename) is False:
                    need_to_restart = True
                copy_if_different(orig_file, target_filename)
    #output += "setup.py is " + str(len(setup_py)) + " characters long\n"
    setup_py = re.sub(r'.*setup\(', '', setup_py, flags=re.DOTALL)
    for line in setup_py.splitlines():
        m = re.search(r"^ *([a-z_]+) *= *\(?'(.*)'", line)
        if m:
            extracted[m.group(1)] = m.group(2)
        m = re.search(r'^ *([a-z_]+) *= *\(?"(.*)"', line)
        if m:
            extracted[m.group(1)] = m.group(2)
        m = re.search(r'^ *([a-z_]+) *= *\[(.*)\]', line)
        if m:
            the_list = []
            for item in re.split(r', *', m.group(2)):
                inner_item = re.sub(r"'$", '', item)
                inner_item = re.sub(r"^'", '', inner_item)
                inner_item = re.sub(r'"+$', '', inner_item)
                inner_item = re.sub(r'^"+', '', inner_item)
                the_list.append(inner_item)
            extracted[m.group(1)] = the_list
    info_dict = dict(readme=readme_text, interview_files=data_files['questions'], sources_files=data_files['sources'], static_files=data_files['static'], module_files=data_files['modules'], template_files=data_files['templates'], dependencies=extracted.get('install_requires', []), description=extracted.get('description', ''), author_name=extracted.get('author', ''), author_email=extracted.get('author_email', ''), license=extracted.get('license', ''), url=extracted.get('url', ''), version=extracted.get('version', ''), github_url=github_url, github_branch=branch, pypi_package_name=pypi_package)
    info_dict['dependencies'] = [x for x in [z for z in map(lambda y: re.sub(r'[\>\<\=].*', '', y), info_dict['dependencies'])] if x not in ('docassemble', 'docassemble.base', 'docassemble.webapp')]
    #output += "info_dict is set\n"
    package_name = re.sub(r'^docassemble\.', '', extracted.get('name', expected_name))
    # if not user_can_edit_package(pkgname='docassemble.' + package_name):
    #     index = 1
    #     orig_package_name = package_name
    #     while index < 100 and not user_can_edit_package(pkgname='docassemble.' + package_name):
    #         index += 1
    #         package_name = orig_package_name + str(index)
    with open(os.path.join(directory_for(area['playgroundpackages'], current_project), 'docassemble.' + package_name), 'w', encoding='utf-8') as fp:
        the_yaml = yaml.safe_dump(info_dict, default_flow_style=False, default_style='|')
        fp.write(str(the_yaml))
    for sec in area:
        area[sec].finalize()
    for key in r.keys('da:interviewsource:docassemble.playground' + str(current_user.id) + ':*'):
        r.incr(key.decode())
    return dict(action='finished', need_to_restart=need_to_restart, package_name=package_name)

def get_github_username_and_email():
    storage = RedisCredStorage(app='github')
    credentials = storage.get()
    if not credentials or credentials.invalid:
        raise Exception('GitHub integration expired.')
    http = credentials.authorize(httplib2.Http())
    resp, content = http.request("https://api.github.com/user", "GET")
    if int(resp['status']) == 200:
        info = json.loads(content.decode('utf-8', 'ignore'))
        github_user_name = info.get('login', None)
        github_author_name = info.get('name', None)
        github_email = info.get('email', None)
    else:
        raise DAError("playground_packages: could not get information about GitHub User")
    if github_email is None:
        resp, content = http.request("https://api.github.com/user/emails", "GET")
        if int(resp['status']) == 200:
            info = json.loads(content.decode('utf-8', 'ignore'))
            for item in info:
                if item.get('email', None) and item.get('visibility', None) != 'private':
                    github_email = item['email']
    if github_user_name is None or github_email is None:
        raise DAError("playground_packages: login not present in user info from GitHub")
    return github_user_name, github_email, github_author_name

@app.route('/playgroundpackages', methods=['GET', 'POST'])
@login_required
@roles_required(['developer', 'admin'])
def playground_packages():
    if not app.config['ENABLE_PLAYGROUND']:
        return ('File not found', 404)
    setup_translation()
    fix_package_folder()
    current_project = get_current_project()
    form = PlaygroundPackagesForm(request.form)
    fileform = PlaygroundUploadForm(request.form)
    the_file = secure_filename_spaces_ok(request.args.get('file', ''))
    no_file_specified = bool(the_file == '')
    scroll = False
    allow_pypi = daconfig.get('pypi', False)
    pypi_username = current_user.pypi_username
    pypi_password = current_user.pypi_password
    pypi_url = daconfig.get('pypi url', 'https://pypi.python.org/pypi')
    can_publish_to_pypi = bool(allow_pypi is True and pypi_username is not None and pypi_password is not None and pypi_username != '' and pypi_password != '')
    if app.config['USE_GITHUB']:
        github_auth = r.get('da:using_github:userid:' + str(current_user.id))
        if github_auth is not None:
            github_auth = github_auth.decode()
            if github_auth == '1':
                github_auth_info = dict(shared=True, orgs=True)
            else:
                github_auth_info = json.loads(github_auth)
            can_publish_to_github = True
        else:
            can_publish_to_github = False
    else:
        can_publish_to_github = None
    if can_publish_to_github and request.method == 'GET':
        storage = RedisCredStorage(app='github')
        credentials = storage.get()
        if not credentials or credentials.invalid:
            state_string = random_string(16)
            session['github_next'] = json.dumps(dict(state=state_string, path='playground_packages', arguments=request.args))
            flow = get_github_flow()
            uri = flow.step1_get_authorize_url(state=state_string)
            return redirect(uri)
    show_message = true_or_false(request.args.get('show_message', True))
    github_message = None
    pypi_message = None
    pypi_version = None
    package_list, package_auth = get_package_info()
    package_names = sorted([package.package.name for package in package_list])
    for default_package in ('docassemble', 'docassemble.base', 'docassemble.webapp'):
        if default_package in package_names:
            package_names.remove(default_package)
    # if the_file:
    #     scroll = True
    if request.method == 'GET':
        is_new = true_or_false(request.args.get('new', False))
    else:
        is_new = False
    if is_new:
        # scroll = True
        the_file = ''
    area = {}
    file_list = {}
    section_name = {'playground': 'Interview files', 'playgroundpackages': 'Packages', 'playgroundtemplate': 'Template files', 'playgroundstatic': 'Static files', 'playgroundsources': 'Source files', 'playgroundmodules': 'Modules'}
    section_sec = {'playgroundtemplate': 'template', 'playgroundstatic': 'static', 'playgroundsources': 'sources', 'playgroundmodules': 'modules'}
    section_field = {'playground': form.interview_files, 'playgroundtemplate': form.template_files, 'playgroundstatic': form.static_files, 'playgroundsources': form.sources_files, 'playgroundmodules': form.module_files}
    for sec in ('playground', 'playgroundpackages', 'playgroundtemplate', 'playgroundstatic', 'playgroundsources', 'playgroundmodules'):
        area[sec] = SavedFile(current_user.id, fix=True, section=sec)
        the_directory = directory_for(area[sec], current_project)
        if sec == 'playground' and current_project != 'default' and not os.path.isdir(the_directory):
            current_project = set_current_project('default')
            the_directory = directory_for(area[sec], current_project)
        file_list[sec] = sorted([f for f in os.listdir(the_directory) if os.path.isfile(os.path.join(the_directory, f)) and re.search(r'^[A-Za-z0-9]', f)])
    for sec, field in section_field.items():
        the_list = []
        for item in file_list[sec]:
            the_list.append((item, item))
        field.choices = the_list
    the_list = []
    for item in package_names:
        the_list.append((item, item))
    form.dependencies.choices = the_list
    validated = False
    form.github_branch.choices = []
    if form.github_branch.data:
        form.github_branch.choices.append((form.github_branch.data, form.github_branch.data))
    else:
        form.github_branch.choices.append(('', ''))
    if request.method == 'POST' and 'uploadfile' not in request.files and form.validate():
        the_file = form.file_name.data
        validated = True
        # else:
            # the_error = ''
            # for attrib in ('original_file_name', 'file_name', 'license', 'description', 'author_name', 'author_email', 'version', 'url', 'dependencies', 'interview_files', 'template_files', 'module_files', 'static_files', 'sources_files', 'readme', 'github_branch', 'commit_message', 'submit', 'download', 'install', 'pypi', 'github', 'cancel', 'delete'):
            #     the_field = getattr(form, attrib)
            #     for error in the_field.errors:
            #         the_error += str(error)
            # raise DAError("Form did not validate with " + str(the_error))
    the_file = re.sub(r'[^A-Za-z0-9\-\_\.]+', '-', the_file)
    the_file = re.sub(r'^docassemble-', r'', the_file)
    the_directory = directory_for(area['playgroundpackages'], current_project)
    files = sorted([f for f in os.listdir(the_directory) if os.path.isfile(os.path.join(the_directory, f)) and re.search(r'^[A-Za-z0-9]', f)])
    editable_files = []
    mode = "yaml"
    for a_file in files:
        editable_files.append(dict(name=re.sub(r'^docassemble.', r'', a_file), modtime=os.path.getmtime(os.path.join(the_directory, a_file))))
    assign_opacity(editable_files)
    editable_file_listing = [x['name'] for x in editable_files]
    if request.method == 'GET' and not the_file and not is_new:
        current_file = get_current_file(current_project, 'packages')
        if not current_file.startswith('docassemble.'):
            current_file = 'docassemble.' + current_file
            set_current_file(current_project, 'packages', current_file)
        if re.sub(r'^docassemble.', r'', current_file) in editable_file_listing:
            the_file = re.sub(r'^docassemble.', r'', current_file)
        else:
            delete_current_file(current_project, 'packages')
            if len(editable_files) > 0:
                the_file = sorted(editable_files, key=lambda x: x['modtime'])[-1]['name']
            else:
                the_file = ''
    #if the_file != '' and not user_can_edit_package(pkgname='docassemble.' + the_file):
    #    flash(word('Sorry, that package name,') + ' ' + the_file + word(', is already in use by someone else'), 'error')
    #    validated = False
    if request.method == 'GET' and the_file in editable_file_listing:
        set_current_file(current_project, 'packages', 'docassemble.' + the_file)
    if the_file == '' and len(file_list['playgroundpackages']) and not is_new:
        the_file = file_list['playgroundpackages'][0]
        the_file = re.sub(r'^docassemble.', r'', the_file)
    old_info = {}
    on_github = False
    branch_info = []
    github_http = None
    github_ssh = None
    github_use_ssh = False
    github_user_name = None
    github_email = None
    github_author_name = None
    github_url_from_file = None
    pypi_package_from_file = None
    expected_name = 'unknown'
    if request.method == 'GET' and the_file != '':
        if the_file != '' and os.path.isfile(os.path.join(directory_for(area['playgroundpackages'], current_project), 'docassemble.' + the_file)):
            filename = os.path.join(directory_for(area['playgroundpackages'], current_project), 'docassemble.' + the_file)
            with open(filename, 'r', encoding='utf-8') as fp:
                content = fp.read()
                old_info = yaml.load(content, Loader=yaml.FullLoader)
                if isinstance(old_info, dict):
                    github_url_from_file = old_info.get('github_url', None)
                    pypi_package_from_file = old_info.get('pypi_package_name', None)
                    for field in ('license', 'description', 'author_name', 'author_email', 'version', 'url', 'readme'):
                        if field in old_info:
                            form[field].data = old_info[field]
                        else:
                            form[field].data = ''
                    if 'dependencies' in old_info and isinstance(old_info['dependencies'], list) and len(old_info['dependencies']):
                        old_info['dependencies'] = [z for z in map(lambda y: re.sub(r'[\>\<\=].*', '', y), old_info['dependencies'])]
                        for item in ('docassemble', 'docassemble.base', 'docassemble.webapp'):
                            if item in old_info['dependencies']:
                                del old_info['dependencies'][item]
                    for field in ('dependencies', 'interview_files', 'template_files', 'module_files', 'static_files', 'sources_files'):
                        if field in old_info and isinstance(old_info[field], list) and len(old_info[field]):
                            form[field].data = old_info[field]
                else:
                    raise Exception("YAML yielded " + repr(old_info) + " from " + repr(content))
        else:
            filename = None
    if the_file != '' and can_publish_to_github and not is_new:
        github_package_name = 'docassemble-' + the_file
        try:
            storage = RedisCredStorage(app='github')
            credentials = storage.get()
            if not credentials or credentials.invalid:
                if form.github.data:
                    state_string = random_string(16)
                    session['github_next'] = json.dumps(dict(state=state_string, path='playground_packages', arguments=request.args))
                    flow = get_github_flow()
                    uri = flow.step1_get_authorize_url(state=state_string)
                    return redirect(uri)
                else:
                    raise Exception('GitHub integration expired.')
            http = credentials.authorize(httplib2.Http())
            resp, content = http.request("https://api.github.com/user", "GET")
            if int(resp['status']) == 200:
                info = json.loads(content.decode('utf-8', 'ignore'))
                github_user_name = info.get('login', None)
                github_author_name = info.get('name', None)
                github_email = info.get('email', None)
            else:
                raise DAError("playground_packages: could not get information about GitHub User")
            if github_email is None:
                resp, content = http.request("https://api.github.com/user/emails", "GET")
                if int(resp['status']) == 200:
                    info = json.loads(content.decode('utf-8', 'ignore'))
                    for item in info:
                        if item.get('email', None) and item.get('visibility', None) != 'private':
                            github_email = item['email']
            if github_user_name is None or github_email is None:
                raise DAError("playground_packages: login not present in user info from GitHub")
            found = False
            found_strong = False
            resp, content = http.request("https://api.github.com/repos/" + str(github_user_name) + "/" + github_package_name, "GET")
            if int(resp['status']) == 200:
                repo_info = json.loads(content.decode('utf-8', 'ignore'))
                github_http = repo_info['html_url']
                github_ssh = repo_info['ssh_url']
                if repo_info['private']:
                    github_use_ssh = True
                github_message = word('This package is') + ' <a target="_blank" href="' + repo_info.get('html_url', 'about:blank') + '">' + word("published on GitHub") + '</a>.'
                if github_author_name:
                    github_message += "  " + word("The author is") + " " + github_author_name + "."
                on_github = True
                branch_info = get_branch_info(http, repo_info['full_name'])
                found = True
                if github_url_from_file is None or github_url_from_file in [github_ssh, github_http]:
                    found_strong = True
            if found_strong is False and github_auth_info['shared']:
                repositories = get_user_repositories(http)
                for repo_info in repositories:
                    if repo_info['name'] != github_package_name or (github_http is not None and github_http == repo_info['html_url']) or (github_ssh is not None and github_ssh == repo_info['ssh_url']):
                        continue
                    if found and github_url_from_file is not None and github_url_from_file not in [repo_info['html_url'], repo_info['ssh_url']]:
                        break
                    github_http = repo_info['html_url']
                    github_ssh = repo_info['ssh_url']
                    if repo_info['private']:
                        github_use_ssh = True
                    github_message = word('This package is') + ' <a target="_blank" href="' + repo_info.get('html_url', 'about:blank') + '">' + word("published on GitHub") + '</a>.'
                    on_github = True
                    branch_info = get_branch_info(http, repo_info['full_name'])
                    found = True
                    if github_url_from_file is None or github_url_from_file in [github_ssh, github_http]:
                        found_strong = True
                    break
            if found_strong is False and github_auth_info['orgs']:
                orgs_info = get_orgs_info(http)
                for org_info in orgs_info:
                    resp, content = http.request("https://api.github.com/repos/" + str(org_info['login']) + "/" + github_package_name, "GET")
                    if int(resp['status']) == 200:
                        repo_info = json.loads(content.decode('utf-8', 'ignore'))
                        if found and github_url_from_file is not None and github_url_from_file not in [repo_info['html_url'], repo_info['ssh_url']]:
                            break
                        github_http = repo_info['html_url']
                        github_ssh = repo_info['ssh_url']
                        if repo_info['private']:
                            github_use_ssh = True
                        github_message = word('This package is') + ' <a target="_blank" href="' + repo_info.get('html_url', 'about:blank') + '">' + word("published on GitHub") + '</a>.'
                        on_github = True
                        branch_info = get_branch_info(http, repo_info['full_name'])
                        found = True
                        if github_url_from_file is None or github_url_from_file in [github_ssh, github_http]:
                            found_strong = True
                        break
            if found is False:
                github_message = word('This package is not yet published on your GitHub account.')
        except Exception as e:
            logmessage('playground_packages: GitHub error.  ' + str(e))
            on_github = None
            github_message = word('Unable to determine if the package is published on your GitHub account.')
    if request.method == 'POST' and 'uploadfile' in request.files:
        the_files = request.files.getlist('uploadfile')
        need_to_restart = False
        if current_user.timezone:
            the_timezone = pytz.timezone(current_user.timezone)
        else:
            the_timezone = pytz.timezone(get_default_timezone())
        epoch_date = datetime.datetime(1970, 1, 1).replace(tzinfo=pytz.utc)
        if the_files:
            for up_file in the_files:
                zip_filename = werkzeug.utils.secure_filename(up_file.filename)
                zippath = tempfile.NamedTemporaryFile(mode="wb", suffix=".zip", prefix="datemp", delete=False)
                up_file.save(zippath.name)
                area_sec = dict(templates='playgroundtemplate', static='playgroundstatic', sources='playgroundsources', questions='playground')
                zippath.close()
                with zipfile.ZipFile(zippath.name, mode='r') as zf:
                    readme_text = ''
                    setup_py = ''
                    extracted = {}
                    data_files = dict(templates=[], static=[], sources=[], interviews=[], modules=[], questions=[])
                    has_docassemble_dir = set()
                    has_setup_file = set()
                    for zinfo in zf.infolist():
                        if zinfo.is_dir():
                            if zinfo.filename.endswith('/docassemble/'):
                                has_docassemble_dir.add(re.sub(r'/docassemble/$', '', zinfo.filename))
                            if zinfo.filename == 'docassemble/':
                                has_docassemble_dir.add('')
                        elif zinfo.filename.endswith('/setup.py'):
                            (directory, filename) = os.path.split(zinfo.filename)
                            has_setup_file.add(directory)
                        elif zinfo.filename == 'setup.py':
                            has_setup_file.add('')
                    root_dir = None
                    for directory in has_docassemble_dir.union(has_setup_file):
                        if root_dir is None or len(directory) < len(root_dir):
                            root_dir = directory
                    if root_dir is None:
                        flash(word("The zip file did not contain a docassemble add-on package."), 'error')
                        return redirect(url_for('playground_packages', project=current_project, file=the_file))
                    for zinfo in zf.infolist():
                        #logmessage("Found a " + zinfo.filename)
                        if zinfo.filename.endswith('/'):
                            continue
                        (directory, filename) = os.path.split(zinfo.filename)
                        if filename.startswith('#') or filename.endswith('~'):
                            continue
                        dirparts = splitall(directory)
                        if '.git' in dirparts:
                            continue
                        levels = re.findall(r'/', directory)
                        time_tuple = zinfo.date_time
                        the_time = time.mktime(datetime.datetime(*time_tuple).timetuple())
                        for sec in ('templates', 'static', 'sources', 'questions'):
                            if directory.endswith('data/' + sec) and filename != 'README.md':
                                data_files[sec].append(filename)
                                target_filename = os.path.join(directory_for(area[area_sec[sec]], current_project), filename)
                                with zf.open(zinfo) as source_fp, open(target_filename, 'wb') as target_fp:
                                    shutil.copyfileobj(source_fp, target_fp)
                                os.utime(target_filename, (the_time, the_time))
                        if filename == 'README.md' and directory == root_dir:
                            with zf.open(zinfo) as f:
                                the_file_obj = TextIOWrapper(f, encoding='utf8')
                                readme_text = the_file_obj.read()
                        if filename == 'setup.py' and directory == root_dir:
                            with zf.open(zinfo) as f:
                                the_file_obj = TextIOWrapper(f, encoding='utf8')
                                setup_py = the_file_obj.read()
                        elif len(levels) >= 2 and filename.endswith('.py') and filename != '__init__.py' and 'tests' not in dirparts and 'data' not in dirparts:
                            need_to_restart = True
                            data_files['modules'].append(filename)
                            target_filename = os.path.join(directory_for(area['playgroundmodules'], current_project), filename)
                            with zf.open(zinfo) as source_fp, open(target_filename, 'wb') as target_fp:
                                shutil.copyfileobj(source_fp, target_fp)
                                os.utime(target_filename, (the_time, the_time))
                    setup_py = re.sub(r'.*setup\(', '', setup_py, flags=re.DOTALL)
                    for line in setup_py.splitlines():
                        m = re.search(r"^ *([a-z_]+) *= *\(?'(.*)'", line)
                        if m:
                            extracted[m.group(1)] = m.group(2)
                        m = re.search(r'^ *([a-z_]+) *= *\(?"(.*)"', line)
                        if m:
                            extracted[m.group(1)] = m.group(2)
                        m = re.search(r'^ *([a-z_]+) *= *\[(.*)\]', line)
                        if m:
                            the_list = []
                            for item in re.split(r', *', m.group(2)):
                                inner_item = re.sub(r"'$", '', item)
                                inner_item = re.sub(r"^'", '', inner_item)
                                inner_item = re.sub(r'"+$', '', inner_item)
                                inner_item = re.sub(r'^"+', '', inner_item)
                                the_list.append(inner_item)
                            extracted[m.group(1)] = the_list
                    info_dict = dict(readme=readme_text, interview_files=data_files['questions'], sources_files=data_files['sources'], static_files=data_files['static'], module_files=data_files['modules'], template_files=data_files['templates'], dependencies=[z for z in map(lambda y: re.sub(r'[\>\<\=].*', '', y), extracted.get('install_requires', []))], description=extracted.get('description', ''), author_name=extracted.get('author', ''), author_email=extracted.get('author_email', ''), license=extracted.get('license', ''), url=extracted.get('url', ''), version=extracted.get('version', ''))

                    info_dict['dependencies'] = [x for x in [z for z in map(lambda y: re.sub(r'[\>\<\=].*', '', y), info_dict['dependencies'])] if x not in ('docassemble', 'docassemble.base', 'docassemble.webapp')]
                    package_name = re.sub(r'^docassemble\.', '', extracted.get('name', expected_name))
                    with open(os.path.join(directory_for(area['playgroundpackages'], current_project), 'docassemble.' + package_name), 'w', encoding='utf-8') as fp:
                        the_yaml = yaml.safe_dump(info_dict, default_flow_style=False, default_style='|')
                        fp.write(str(the_yaml))
                    for key in r.keys('da:interviewsource:docassemble.playground' + str(current_user.id) + project_name(current_project) + ':*'):
                        r.incr(key.decode())
                    for sec in area:
                        area[sec].finalize()
                    the_file = package_name
                zippath.close()
        if show_message:
            flash(word("The package was unpacked into the Playground."), 'success')
        if need_to_restart:
            return redirect(url_for('restart_page', next=url_for('playground_packages', project=current_project, file=the_file)))
        return redirect(url_for('playground_packages', project=current_project, file=the_file))
    if request.method == 'GET' and 'pull' in request.args and int(request.args['pull']) == 1 and ('github_url' in request.args or 'pypi' in request.args):
        if can_publish_to_github and (github_user_name is None or github_email is None):
            (github_user_name, github_email, github_author_name) = get_github_username_and_email()
        github_url = request.args.get('github_url', None)
        pypi_package = request.args.get('pypi', None)
        branch = request.args.get('branch', None)
        do_pypi_also = true_or_false(request.args.get('pypi_also', False))
        do_install_also = true_or_false(request.args.get('install_also', False))
        result = do_playground_pull(area, current_project, github_url=github_url, branch=branch, pypi_package=pypi_package, can_publish_to_github=can_publish_to_github, github_email=github_email, pull_only=(do_pypi_also or do_install_also))
        if result['action'] == 'error':
            raise DAError("playground_packages: " + result['message'])
        if result['action'] == 'fail':
            flash(result['message'], 'error')
            return redirect(url_for('playground_packages', project=current_project))
        if result['action'] == 'pull_only':
            the_args = dict(package=the_file, project=current_project)
            if do_pypi_also:
                the_args['pypi'] = '1'
            if do_install_also:
                the_args['install'] = '1'
            area['playgroundpackages'].finalize()
            return redirect(url_for('create_playground_package', **the_args))
        if result['action'] == 'finished':
            the_file = result['package_name']
            if show_message:
                flash(word("The package was unpacked into the Playground."), 'success')
            #shutil.rmtree(directory)
            if result['need_to_restart']:
                return redirect(url_for('restart_page', next=url_for('playground_packages', file=the_file, project=current_project)))
            return redirect(url_for('playground_packages', project=current_project, file=the_file))
    if request.method == 'POST' and validated and form.delete.data and the_file != '' and the_file == form.file_name.data and os.path.isfile(os.path.join(directory_for(area['playgroundpackages'], current_project), 'docassemble.' + the_file)):
        os.remove(os.path.join(directory_for(area['playgroundpackages'], current_project), 'docassemble.' + the_file))
        dotfile = os.path.join(directory_for(area['playgroundpackages'], current_project), '.docassemble-' + the_file)
        if os.path.exists(dotfile):
            os.remove(dotfile)
        area['playgroundpackages'].finalize()
        flash(word("Deleted package"), "success")
        return redirect(url_for('playground_packages', project=current_project))
    if not is_new:
        pkgname = 'docassemble.' + the_file
        pypi_info = pypi_status(pkgname)
        if can_publish_to_pypi:
            if pypi_info['error']:
                pypi_message = word("Unable to determine if the package is published on PyPI.")
            else:
                if pypi_info['exists'] and 'info' in pypi_info['info']:
                    pypi_version = pypi_info['info']['info'].get('version', None)
                    pypi_message = word('This package is') + ' <a target="_blank" href="' + pypi_url + '/' + pkgname + '/' + pypi_version + '">' + word("published on PyPI") + '</a>.'
                    pypi_author = pypi_info['info']['info'].get('author', None)
                    if pypi_author:
                        pypi_message += "  " + word("The author is") + " " + pypi_author + "."
                    if pypi_version != form['version'].data:
                        pypi_message += "  " + word("The version on PyPI is") + " " + str(pypi_version) + ".  " + word("Your version is") + " " + str(form['version'].data) + "."
                else:
                    pypi_message = word('This package is not yet published on PyPI.')
    if request.method == 'POST' and validated:
        new_info = {}
        for field in ('license', 'description', 'author_name', 'author_email', 'version', 'url', 'readme', 'dependencies', 'interview_files', 'template_files', 'module_files', 'static_files', 'sources_files'):
            new_info[field] = form[field].data
        #logmessage("found " + str(new_info))
        if form.submit.data or form.download.data or form.install.data or form.pypi.data or form.github.data:
            if the_file != '':
                area['playgroundpackages'].finalize()
                if form.original_file_name.data and form.original_file_name.data != the_file:
                    old_filename = os.path.join(directory_for(area['playgroundpackages'], current_project), 'docassemble.' + form.original_file_name.data)
                    if os.path.isfile(old_filename):
                        os.remove(old_filename)
                if form.pypi.data and pypi_version is not None:
                    if not new_info['version']:
                        new_info['version'] = pypi_version
                    while 'releases' in pypi_info['info'] and new_info['version'] in pypi_info['info']['releases'].keys():
                        versions = new_info['version'].split(".")
                        versions[-1] = str(int(versions[-1]) + 1)
                        new_info['version'] = ".".join(versions)
                filename = os.path.join(directory_for(area['playgroundpackages'], current_project), 'docassemble.' + the_file)
                if os.path.isfile(filename):
                    with open(filename, 'r', encoding='utf-8') as fp:
                        content = fp.read()
                        old_info = yaml.load(content, Loader=yaml.FullLoader)
                    for name in ('github_url', 'github_branch', 'pypi_package_name'):
                        if old_info.get(name, None):
                            new_info[name] = old_info[name]
                with open(filename, 'w', encoding='utf-8') as fp:
                    the_yaml = yaml.safe_dump(new_info, default_flow_style=False, default_style = '|')
                    fp.write(str(the_yaml))
                area['playgroundpackages'].finalize()
                if form.download.data:
                    return redirect(url_for('create_playground_package', package=the_file, project=current_project))
                if form.install.data:
                    return redirect(url_for('create_playground_package', package=the_file, project=current_project, install='1'))
                if form.pypi.data:
                    if form.install_also.data:
                        return redirect(url_for('create_playground_package', package=the_file, project=current_project, pypi='1', install='1'))
                    else:
                        return redirect(url_for('create_playground_package', package=the_file, project=current_project, pypi='1'))
                if form.github.data:
                    the_branch = form.github_branch.data
                    if the_branch == "<new>":
                        the_branch = re.sub(r'[^A-Za-z0-9\_\-]', r'', str(form.github_branch_new.data))
                        return redirect(url_for('create_playground_package', project=current_project, package=the_file, github='1', commit_message=form.commit_message.data, new_branch=str(the_branch), pypi_also=('1' if form.pypi_also.data else '0'), install_also=('1' if form.install_also.data else '0')))
                    else:
                        return redirect(url_for('create_playground_package', project=current_project, package=the_file, github='1', commit_message=form.commit_message.data, branch=str(the_branch), pypi_also=('1' if form.pypi_also.data else '0'), install_also=('1' if form.install_also.data else '0')))
                the_time = formatted_current_time()
                if show_message:
                    flash(word('The package information was saved.'), 'success')
    form.original_file_name.data = the_file
    form.file_name.data = the_file
    if the_file != '' and os.path.isfile(os.path.join(directory_for(area['playgroundpackages'], current_project), 'docassemble.' + the_file)):
        filename = os.path.join(directory_for(area['playgroundpackages'], current_project), 'docassemble.' + the_file)
    else:
        filename = None
    header = word("Packages")
    upload_header = None
    edit_header = None
    description = Markup("""Describe your package and choose the files from your Playground that will go into it.""")
    after_text = None
    if scroll:
        extra_command = "        scrollBottom();"
    else:
        extra_command = ""
    extra_command += upload_js() + """
        $("#daCancelPyPI").click(function(event){
          var daWhichButton = this;
          $("#pypi_message_div").hide();
          $(".btn-da").each(function(){
            if (this != daWhichButton && $(this).attr('id') != 'daCancelGitHub' && $(this).is(":hidden")){
              $(this).show();
            }
          });
          $("#daPyPI").html(""" + json.dumps(word("PyPI")) + """);
          $(this).hide();
          event.preventDefault();
          return false;
        });
        $("#daCancelGitHub").click(function(event){
          var daWhichButton = this;
          $("#commit_message_div").hide();
          $(".btn-da").each(function(){
            if (this != daWhichButton && $(this).attr('id') != 'daCancelPyPI' && $(this).is(":hidden")){
              $(this).show();
            }
          });
          $("#daGitHub").html(""" + json.dumps(word("GitHub")) + """);
          $(this).hide();
          event.preventDefault();
          return false;
        });
        if ($("#github_branch option").length == 0){
          $("#github_branch_div").hide();
        }
        $("#github_branch").on('change', function(event){
          if ($(this).val() == '<new>'){
            $("#new_branch_div").show();
          }
          else{
            $("#new_branch_div").hide();
          }
        });
        $("#daPyPI").click(function(event){
          if (existingPypiVersion != null && existingPypiVersion == $("#version").val()){
            alert(""" + json.dumps(word("You need to increment the version before publishing to PyPI.")) + """);
            $('html, body').animate({
              scrollTop: $("#version").offset().top-90,
              scrollLeft: 0
            });
            $("#version").focus();
            var tmpStr = $("#version").val();
            $("#version").val('');
            $("#version").val(tmpStr);
            event.preventDefault();
            return false;
          }
          var daWhichButton = this;
          if ($("#pypi_message_div").is(":hidden")){
            $("#pypi_message_div").show();
            $(".btn-da").each(function(){
              if (this != daWhichButton && $(this).is(":visible")){
                $(this).hide();
              }
            });
            $(this).html(""" + json.dumps(word("Publish")) + """);
            $("#daCancelPyPI").show();
            window.scrollTo(0, document.body.scrollHeight);
            event.preventDefault();
            return false;
          }
        });
        $("#daGitHub").click(function(event){
          var daWhichButton = this;
          if ($("#commit_message").val().length == 0 || $("#commit_message_div").is(":hidden")){
            if ($("#commit_message_div").is(":visible")){
              $("#commit_message").addClass("is-invalid");
            }
            else{
              $("#commit_message_div").show();
              $(".btn-da").each(function(){
                if (this != daWhichButton && $(this).is(":visible")){
                  $(this).hide();
                }
              });
              $(this).html(""" + json.dumps(word("Commit")) + """);
              $("#daCancelGitHub").show();
            }
            $("#commit_message").focus();
            window.scrollTo(0, document.body.scrollHeight);
            event.preventDefault();
            return false;
          }
          if ($("#pypi_also").prop('checked') && existingPypiVersion != null && existingPypiVersion == $("#version").val()){
            alert(""" + json.dumps(word("You need to increment the version before publishing to PyPI.")) + """);
            $('html, body').animate({
              scrollTop: $("#version").offset().top-90,
              scrollLeft: 0
            });
            $("#version").focus();
            var tmpStr = $("#version").val();
            $("#version").val('');
            $("#version").val(tmpStr);
            event.preventDefault();
            return false;
          }
        });
        $(document).on('keydown', function(e){
          if (e.which == 13){
            var tag = $( document.activeElement ).prop("tagName");
            if (tag != "TEXTAREA" && tag != "A" && tag != "LABEL" && tag != "BUTTON"){
              e.preventDefault();
              e.stopPropagation();
            }
          }
        });"""
    if keymap:
        kbOpt = 'keyMap: "' + keymap + '", cursorBlinkRate: 0, '
        kbLoad = '<script src="' + url_for('static', filename="codemirror/keymap/" + keymap + ".js", v=da_version) + '"></script>\n    '
    else:
        kbOpt = ''
        kbLoad = ''
    any_files = len(editable_files) > 0
    back_button = Markup('<span class="navbar-brand navbar-nav dabackicon me-3"><a href="' + url_for('playground_page', project=current_project) + '" class="dabackbuttoncolor nav-link" title=' + json.dumps(word("Go back to the main Playground page")) + '><i class="fas fa-chevron-left"></i><span class="daback">' + word('Back') + '</span></a></span>')
    if can_publish_to_pypi:
        if pypi_message is not None:
            pypi_message = Markup(pypi_message)
    else:
        pypi_message = None
    extra_js = '\n    <script src="' + url_for('static', filename="app/playgroundbundle.js", v=da_version) + '"></script>\n    '
    extra_js += kbLoad
    extra_js += """<script>
      var existingPypiVersion = """ + json.dumps(pypi_version) + """;
      var isNew = """ + json.dumps(is_new) + """;
      var existingFiles = """ + json.dumps(files) + """;
      var currentFile = """ + json.dumps(the_file) + """;
      var daExpireSession = null;
      function resetExpireSession(){
        if (daExpireSession != null){
          window.clearTimeout(daExpireSession);
        }
        daExpireSession = setTimeout(function(){
          alert(""" + json.dumps(word("Your browser session has expired and you have been signed out.  You will not be able to save your work.  Please log in again.")) + """);
        }, """ + str(999 * int(daconfig.get('session lifetime seconds', 43200))) + """);
      }
      function scrollBottom(){
        $("html, body").animate({ scrollTop: $(document).height() }, "slow");
      }
      $( document ).ready(function() {
        resetExpireSession();
        $("#file_name").on('change', function(){
          var newFileName = $(this).val();
          if ((!isNew) && newFileName == currentFile){
            return;
          }
          for (var i = 0; i < existingFiles.length; i++){
            if (newFileName == existingFiles[i]){
              alert(""" + json.dumps(word("Warning: a package definition by that name already exists.  If you save, you will overwrite it.")) + """);
              return;
            }
          }
          return;
        });
        $("#daDelete").click(function(event){
          if (!confirm(""" + '"' + word("Are you sure that you want to delete this package?") + '"' + """)){
            event.preventDefault();
          }
        });
        daTextArea = document.getElementById("readme");
        var daCodeMirror = CodeMirror.fromTextArea(daTextArea, {mode: "markdown", """ + kbOpt + """tabSize: 2, tabindex: 70, autofocus: false, lineNumbers: true, matchBrackets: true, lineWrapping: """ + ('true' if daconfig.get('wrap lines in playground', True) else 'false') + """});
        $(window).bind("beforeunload", function(){
          daCodeMirror.save();
          $("#form").trigger("checkform.areYouSure");
        });
        $("#form").areYouSure(""" + json.dumps({'message': word("There are unsaved changes.  Are you sure you wish to leave this page?")}) + """);
        $("#form").bind("submit", function(){
          daCodeMirror.save();
          $("#form").trigger("reinitialize.areYouSure");
          return true;
        });
        daCodeMirror.setOption("extraKeys", { Tab: function(cm){ var spaces = Array(cm.getOption("indentUnit") + 1).join(" "); cm.replaceSelection(spaces); }, "F11": function(cm) { cm.setOption("fullScreen", !cm.getOption("fullScreen")); }, "Esc": function(cm) { if (cm.getOption("fullScreen")) cm.setOption("fullScreen", false); }});
        daCodeMirror.setOption("coverGutterNextToScrollbar", true);""" + extra_command + """
      });
    </script>"""
    if github_use_ssh:
        the_github_url = github_ssh
    else:
        the_github_url = github_http
    if the_github_url is None and github_url_from_file is not None:
        the_github_url = github_url_from_file
    if the_github_url is None:
        the_pypi_package_name = pypi_package_from_file
    else:
        the_pypi_package_name = None
    if github_message is not None and github_url_from_file is not None and github_url_from_file != github_http and github_url_from_file != github_ssh:
        github_message += '  ' + word("This package was originally pulled from") + ' <a target="_blank" href="' + github_as_http(github_url_from_file) + '">' + word('a GitHub repository') + '</a>.'
    if github_message is not None and old_info.get('github_branch', None) and (github_http or github_url_from_file):
        html_url = github_http or github_url_from_file
        commit_code = None
        current_commit_file = os.path.join(directory_for(area['playgroundpackages'], current_project), '.' + github_package_name)
        if os.path.isfile(current_commit_file):
            with open(current_commit_file, 'r', encoding='utf-8') as fp:
                commit_code = fp.read().strip()
            if current_user.timezone:
                the_timezone = pytz.timezone(current_user.timezone)
            else:
                the_timezone = pytz.timezone(get_default_timezone())
            commit_code_date = datetime.datetime.utcfromtimestamp(os.path.getmtime(current_commit_file)).replace(tzinfo=pytz.utc).astimezone(the_timezone).strftime("%Y-%m-%d %H:%M:%S %Z")
        if commit_code:
            github_message += '  ' + word('The current branch is %s and the current commit is %s.') % ('<a target="_blank" href="' + html_url + '/tree/' + old_info['github_branch'] + '">' + old_info['github_branch'] + '</a>', '<a target="_blank" href="' + html_url + '/commit/' + commit_code + '"><code>' + commit_code[0:7] + '</code></a>') + '  ' + word('The commit was saved locally at %s.') % commit_code_date
        else:
            github_message += '  ' + word('The current branch is %s.') % ('<a target="_blank" href="' + html_url + '/tree/' + old_info['github_branch'] + '">' + old_info['github_branch'] + '</a>',)
    if github_message is not None:
        github_message = Markup(github_message)
    branch = old_info.get('github_branch', None)
    if branch is not None:
        branch = branch.strip()
    branch_choices = []
    if len(branch_info) > 0:
        branch_choices.append(("<new>", word("(New branch)")))
    branch_names = set()
    for br in branch_info:
        branch_names.add(br['name'])
        branch_choices.append((br['name'], br['name']))
    if branch and branch in branch_names:
        form.github_branch.data = branch
        default_branch = branch
    elif 'master' in branch_names:
        form.github_branch.data = 'master'
        default_branch = 'master'
    elif 'main' in branch_names:
        form.github_branch.data = 'main'
        default_branch = 'main'
    else:
        default_branch = GITHUB_BRANCH
    form.github_branch.choices = branch_choices
    if form.author_name.data in ('', None) and current_user.first_name and current_user.last_name:
        form.author_name.data = current_user.first_name + " " + current_user.last_name
    if form.author_email.data in ('', None) and current_user.email:
        form.author_email.data = current_user.email
    if current_project != 'default':
        header += " / " + current_project
    response = make_response(render_template('pages/playgroundpackages.html', current_project=current_project, branch=default_branch, version_warning=None, bodyclass='daadminbody', can_publish_to_pypi=can_publish_to_pypi, pypi_message=pypi_message, can_publish_to_github=can_publish_to_github, github_message=github_message, github_url=the_github_url, pypi_package_name=the_pypi_package_name, back_button=back_button, tab_title=header, page_title=header, extra_css=Markup('\n    <link href="' + url_for('static', filename='app/playgroundbundle.css', v=da_version) + '" rel="stylesheet">'), extra_js=Markup(extra_js), header=header, upload_header=upload_header, edit_header=edit_header, description=description, form=form, fileform=fileform, files=files, file_list=file_list, userid=current_user.id, editable_files=sorted(editable_files, key=lambda y: y['name'].lower()), current_file=the_file, after_text=after_text, section_name=section_name, section_sec=section_sec, section_field=section_field, package_names=sorted(package_names, key=lambda y: y.lower()), any_files=any_files), 200)
    response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
    return response

def github_as_http(url):
    if url.startswith('http'):
        return url
    return re.sub('^[^@]+@([^:]+):(.*)\.git$', r'https://\1/\2', url)

def copy_if_different(source, destination):
    if (not os.path.isfile(destination)) or filecmp.cmp(source, destination) is False:
        shutil.copyfile(source, destination)

def splitall(path):
    allparts = []
    while 1:
        parts = os.path.split(path)
        if parts[0] == path:
            allparts.insert(0, parts[0])
            break
        elif parts[1] == path:
            allparts.insert(0, parts[1])
            break
        else:
            path = parts[0]
            allparts.insert(0, parts[1])
    return allparts

@app.route('/playground_redirect_poll', methods=['GET'])
@login_required
@roles_required(['developer', 'admin'])
def playground_redirect_poll():
    if not app.config['ENABLE_PLAYGROUND']:
        return ('File not found', 404)
    key = 'da:runplayground:' + str(current_user.id)
    the_url = r.get(key)
    #logmessage("playground_redirect: key " + str(key) + " is " + str(the_url))
    if the_url is not None:
        the_url = the_url.decode()
        r.delete(key)
        return jsonify(dict(success=True, url=the_url))
    return jsonify(dict(success=False, url=the_url))

@app.route('/playground_redirect', methods=['GET', 'POST'])
@login_required
@roles_required(['developer', 'admin'])
def playground_redirect():
    if not app.config['ENABLE_PLAYGROUND']:
        return ('File not found', 404)
    key = 'da:runplayground:' + str(current_user.id)
    counter = 0
    while counter < 15:
        the_url = r.get(key)
        #logmessage("playground_redirect: key " + str(key) + " is " + str(the_url))
        if the_url is not None:
            the_url = the_url.decode()
            r.delete(key)
            return redirect(the_url)
        time.sleep(1)
        counter += 1
    return ('File not found', 404)

def upload_js():
    return """
        $("#uploadlink").on('click', function(event){
          $("#uploadlabel").click();
          event.preventDefault();
          return false;
        });
        $("#uploadlabel").on('click', function(event){
          event.stopPropagation();
          event.preventDefault();
          $("#uploadfile").click();
          return false;
        });
        $("#uploadfile").on('click', function(event){
          event.stopPropagation();
        });
        $("#uploadfile").on('change', function(event){
          $("#fileform").submit();
        });"""

def search_js(form=None):
    if form is None:
        form = 'form'
    return """
var origPosition = null;
var searchMatches = null;

function searchReady(){
  $("#""" + form + """ input[name='search_term']").on("focus", function(event){
    origPosition = daCodeMirror.getCursor('from');
  });
  $("#""" + form + """ input[name='search_term']").change(update_search);
  $("#""" + form + """ input[name='search_term']").on("keydown", enter_search);
  $("#""" + form + """ input[name='search_term']").on("keyup", update_search);
  $("#daSearchPrevious").click(function(event){
    var query = $("#""" + form + """ input[name='search_term']").val();
    if (query.length == 0){
      clear_matches();
      daCodeMirror.setCursor(daCodeMirror.getCursor('from'));
      $("#""" + form + """ input[name='search_term']").removeClass("da-search-error");
      return;
    }
    origPosition = daCodeMirror.getCursor('from');
    var sc = daCodeMirror.getSearchCursor(query, origPosition);
    show_matches(query);
    var found = sc.findPrevious();
    if (found){
      daCodeMirror.setSelection(sc.from(), sc.to());
      scroll_to_selection();
      $("#""" + form + """ input[name='search_term']").removeClass("da-search-error");
    }
    else{
      var lastLine = daCodeMirror.lastLine()
      var lastChar = daCodeMirror.lineInfo(lastLine).text.length
      origPosition = { line: lastLine, ch: lastChar, xRel: 1 }
      sc = daCodeMirror.getSearchCursor(query, origPosition);
      show_matches(query);
      var found = sc.findPrevious();
      if (found){
        daCodeMirror.setSelection(sc.from(), sc.to());
        scroll_to_selection();
        $("#""" + form + """ input[name='search_term']").removeClass("da-search-error");
      }
      else{
        $("#""" + form + """ input[name='search_term']").addClass("da-search-error");
      }
    }
    event.preventDefault();
    return false;
  });
  $("#daSearchNext").click(function(event){
    var query = $("#""" + form + """ input[name='search_term']").val();
    if (query.length == 0){
      clear_matches();
      daCodeMirror.setCursor(daCodeMirror.getCursor('from'));
      $("#""" + form + """ input[name='search_term']").removeClass("da-search-error");
      return;
    }
    origPosition = daCodeMirror.getCursor('to');
    var sc = daCodeMirror.getSearchCursor(query, origPosition);
    show_matches(query);
    var found = sc.findNext();
    if (found){
      daCodeMirror.setSelection(sc.from(), sc.to());
      scroll_to_selection();
      $("#""" + form + """ input[name='search_term']").removeClass("da-search-error");
    }
    else{
      origPosition = { line: 0, ch: 0, xRel: 1 }
      sc = daCodeMirror.getSearchCursor(query, origPosition);
      show_matches(query);
      var found = sc.findNext();
      if (found){
        daCodeMirror.setSelection(sc.from(), sc.to());
        scroll_to_selection();
        $("#""" + form + """ input[name='search_term']").removeClass("da-search-error");
      }
      else{
        $("#""" + form + """ input[name='search_term']").addClass("da-search-error");
      }
    }
    event.preventDefault();
    return false;
  });
}

function show_matches(query){
  clear_matches();
  if (query.length == 0){
    daCodeMirror.setCursor(daCodeMirror.getCursor('from'));
    $("#""" + form + """ input[name='search_term']").removeClass("da-search-error");
    return;
  }
  searchMatches = daCodeMirror.showMatchesOnScrollbar(query);
}

function clear_matches(){
  if (searchMatches != null){
    try{
      searchMatches.clear();
    }
    catch(err){}
  }
}

function scroll_to_selection(){
  daCodeMirror.scrollIntoView(daCodeMirror.getCursor('from'))
  var t = daCodeMirror.charCoords(daCodeMirror.getCursor('from'), "local").top;
  daCodeMirror.scrollTo(null, t);
}

function enter_search(event){
  var theCode = event.which || event.keyCode;
  if(theCode == 13) {
    event.preventDefault();
    $("#daSearchNext").click();
    return false;
  }
}

function update_search(event){
  var query = $(this).val();
  if (query.length == 0){
    clear_matches();
    daCodeMirror.setCursor(daCodeMirror.getCursor('from'));
    $(this).removeClass("da-search-error");
    return;
  }
  var theCode = event.which || event.keyCode;
  if(theCode == 13) {
    event.preventDefault();
    return false;
  }
  var sc = daCodeMirror.getSearchCursor(query, origPosition);
  show_matches(query);

  var found = sc.findNext();
  if (found){
    daCodeMirror.setSelection(sc.from(), sc.to());
    scroll_to_selection();
    $(this).removeClass("da-search-error");
  }
  else{
    origPosition = { line: 0, ch: 0, xRel: 1 }
    sc = daCodeMirror.getSearchCursor(query, origPosition);
    show_matches(query);
    var found = sc.findNext();
    if (found){
      daCodeMirror.setSelection(sc.from(), sc.to());
      scroll_to_selection();
      $(this).removeClass("da-search-error");
    }
    else{
      $(this).addClass("da-search-error");
    }
  }
}

"""

def variables_js(form=None, office_mode=False):
    output = """
function activatePopovers(){
  var daPopoverTriggerList = [].slice.call(document.querySelectorAll('[data-bs-toggle="popover"]'));
  var daPopoverList = daPopoverTriggerList.map(function (daPopoverTriggerEl) {
    return new bootstrap.Popover(daPopoverTriggerEl, {trigger: "focus", html: true});
  });
}

function activateVariables(){
  $(".daparenthetical").on("click", function(event){
    var reference = $(this).data("ref");
    //console.log("reference is " + reference);
    var target = $('[data-name="' + reference + '"]').first();
    if (target.length > 0){
      //console.log("target is " + target);
      //console.log("scrolltop is now " + $('#daplaygroundcard').scrollTop());
      //console.log("Scrolling to " + target.parent().parent().position().top);
      $('#daplaygroundcard').animate({
          scrollTop: target.parent().parent().position().top
      }, 1000);
    }
    event.preventDefault();
  });

  $(".dashowmethods").on("click", function(event){
    var target_id = $(this).data("showhide");
    $("#" + target_id).slideToggle();
  });

  $(".dashowattributes").each(function(){
    var basename = $(this).data('name');
    if (attrs_showing.hasOwnProperty(basename)){
      if (attrs_showing[basename]){
        $('tr[data-parent="' + basename + '"]').show();
      }
    }
    else{
      attrs_showing[basename] = false;
    }
  });

  $(".dashowattributes").on("click", function(event){
    var basename = $(this).data('name');
    attrs_showing[basename] = !attrs_showing[basename];
    $('tr[data-parent="' + basename + '"]').each(function(){
      $(this).toggle();
    });
  });"""
    if office_mode:
        return output + "\n}"
    if form is None:
        form = 'form'
    output += """
  $(".playground-variable").on("click", function(event){
    daCodeMirror.replaceSelection($(this).data("insert"), "around");
    daCodeMirror.focus();
  });

  $(".dasearchicon").on("click", function(event){
    var query = $(this).data('name');
    if (query == null || query.length == 0){
      clear_matches();
      daCodeMirror.setCursor(daCodeMirror.getCursor('from'));
      return;
    }
    origPosition = daCodeMirror.getCursor('to');
    $("#""" + form + """ input[name='search_term']").val(query);
    var sc = daCodeMirror.getSearchCursor(query, origPosition);
    show_matches(query);
    var found = sc.findNext();
    if (found){
      daCodeMirror.setSelection(sc.from(), sc.to());
      scroll_to_selection();
      $("#form input[name='search_term']").removeClass('da-search-error');
    }
    else{
      origPosition = { line: 0, ch: 0, xRel: 1 }
      sc = daCodeMirror.getSearchCursor(query, origPosition);
      show_matches(query);
      var found = sc.findNext();
      if (found){
        daCodeMirror.setSelection(sc.from(), sc.to());
        scroll_to_selection();
        $("#""" + form + """ input[name='search_term']").removeClass('da-search-error');
      }
      else{
        $("#""" + form + """ input[name='search_term']").addClass('da-search-error');
      }
    }
    event.preventDefault();
    return false;
  });
}

var interviewBaseUrl = '""" + url_for('index', reset='1', cache='0', i='docassemble.playground' + str(current_user.id) + ':.yml') + """';
var shareBaseUrl = '""" + url_for('index', i='docassemble.playground' + str(current_user.id) + ':.yml') + """';

function updateRunLink(){
  if (currentProject == 'default'){
    $("#daRunButton").attr("href", interviewBaseUrl.replace('%3A.yml', ':' + $("#daVariables").val()));
    $("a.da-example-share").attr("href", shareBaseUrl.replace('%3A.yml', ':' + $("#daVariables").val()));
  }
  else{
    $("#daRunButton").attr("href", interviewBaseUrl.replace('%3A.yml', currentProject + ':' + $("#daVariables").val()));
    $("a.da-example-share").attr("href", shareBaseUrl.replace('%3A.yml', currentProject + ':' + $("#daVariables").val()));
  }
}

function fetchVars(changed){
  daCodeMirror.save();
  updateRunLink();
  $.ajax({
    type: "POST",
    url: """ + '"' + url_for('playground_variables') + '"' + """ + '?project=' + currentProject,
    data: 'csrf_token=' + $("#""" + form + """ input[name='csrf_token']").val() + '&variablefile=' + $("#daVariables").val() + '&ajax=1&changed=' + (changed ? 1 : 0),
    success: function(data){
      if (data.action && data.action == 'reload'){
        location.reload(true);
      }
      if (data.vocab_list != null){
        vocab = data.vocab_list;
      }
      if (data.current_project != null){
        currentProject = data.current_project;
      }
      if (data.variables_html != null){
        $("#daplaygroundtable").html(data.variables_html);
        var daPopoverTriggerList = [].slice.call(document.querySelectorAll('[data-bs-toggle="popover"]'));
        var daPopoverList = daPopoverTriggerList.map(function (daPopoverTriggerEl) {
          return new bootstrap.Popover(daPopoverTriggerEl, {trigger: "focus", html: true});
        });
        activateVariables();
      }
    },
    dataType: 'json'
  });
  $("#daVariables").blur();
}

function variablesReady(){
  $("#daVariables").change(function(event){
    fetchVars(true);
  });
}

$( document ).ready(function() {
  $(document).on('keydown', function(e){
    if (e.which == 13){
      var tag = $( document.activeElement ).prop("tagName");
      if (tag == "INPUT"){
        e.preventDefault();
        e.stopPropagation();
        $(".CodeMirror textarea").focus();
        return false;
      }
    }
  });
});
"""
    return output

@app.route("/varsreport", methods=['GET'])
@login_required
@roles_required(['admin', 'developer'])
def variables_report():
    if not app.config['ENABLE_PLAYGROUND']:
        return ('File not found', 404)
    setup_translation()
    playground = SavedFile(current_user.id, fix=True, section='playground')
    the_file = request.args.get('file', None)
    if the_file is not None:
        the_file = secure_filename_spaces_ok(the_file)
    current_project = werkzeug.utils.secure_filename(request.args.get('project', 'default'))
    the_directory = directory_for(playground, current_project)
    files = sorted([f for f in os.listdir(the_directory) if os.path.isfile(os.path.join(the_directory, f)) and re.search(r'^[A-Za-z0-9]', f)])
    if len(files) == 0:
        return jsonify(success=False, reason=1)
    if the_file is None or the_file not in files:
        return jsonify(success=False, reason=2)
    interview_source = docassemble.base.parse.interview_source_from_string('docassemble.playground' + str(current_user.id) + project_name(current_project) + ':' + the_file)
    interview_source.set_testing(True)
    interview = interview_source.get_interview()
    ensure_ml_file_exists(interview, the_file, current_project)
    yaml_file = 'docassemble.playground' + str(current_user.id) + project_name(current_project) + ':' + the_file
    interview_status = docassemble.base.parse.InterviewStatus(current_info=current_info(yaml=yaml_file, req=request, action=None, device_id=request.cookies.get('ds', None)))
    variables_html, vocab_list, vocab_dict = get_vars_in_use(interview, interview_status, debug_mode=False, current_project=current_project)
    results = []
    result_dict = {}
    for name in vocab_list:
        if name in ('x', 'row_item', 'i', 'j', 'k', 'l', 'm', 'n') or name.startswith('x.') or name.startswith('x[') or name.startswith('row_item.'):
            continue
        result = dict(name=name, questions=[])
        results.append(result)
        result_dict[name] = result
    for question in interview.questions_list:
        names_seen = {}
        for the_type, the_set in (('in mako', question.mako_names), ('mentioned in', question.names_used), ('defined by', question.fields_used)):
            for name in the_set:
                the_name = name
                subnames = [the_name]
                while True:
                    if re.search(r'\[[^\]]\]$', the_name):
                        the_name = re.sub(r'\[[^\]]\]$', '', the_name)
                    elif '.' in the_name:
                        the_name = re.sub(r'\.[^\.]*$', '', the_name)
                    else:
                        break
                    subnames.append(the_name)
                on_first = True
                for subname in subnames:
                    if the_type == 'defined by' and not on_first:
                        the_type = 'mentioned in'
                    on_first = False
                    if subname not in result_dict:
                        continue
                    if subname not in names_seen:
                        names_seen[subname] = dict(yaml_file=question.from_source.path, source_code=question.source_code.strip(), usage=[])
                        result_dict[subname]['questions'].append(names_seen[subname])
                    if the_type not in names_seen[subname]['usage']:
                        names_seen[subname]['usage'].append(the_type)
    return jsonify(success=True, yaml_file=yaml_file, items=results)

@app.route('/playgroundvariables', methods=['POST'])
@login_required
@roles_required(['developer', 'admin'])
def playground_variables():
    current_project = get_current_project()
    if not app.config['ENABLE_PLAYGROUND']:
        return ('File not found', 404)
    setup_translation()
    playground = SavedFile(current_user.id, fix=True, section='playground')
    the_directory = directory_for(playground, current_project)
    files = sorted([f for f in os.listdir(the_directory) if os.path.isfile(os.path.join(the_directory, f)) and re.search(r'^[A-Za-z0-9]', f)])
    if len(files) == 0:
        return jsonify(success=False, reason=1)
    post_data = request.form.copy()
    if request.method == 'POST' and 'variablefile' in post_data:
        active_file = post_data['variablefile']
        if post_data['variablefile'] in files:
            if 'changed' in post_data and int(post_data['changed']):
                set_variable_file(current_project, active_file)
            interview_source = docassemble.base.parse.interview_source_from_string('docassemble.playground' + str(current_user.id) + project_name(current_project) + ':' + active_file)
            interview_source.set_testing(True)
        else:
            if active_file == '':
                active_file = 'test.yml'
            content = ''
            if 'playground_content' in post_data:
                content = re.sub(r'\r\n', r'\n', post_data['playground_content'])
            interview_source = docassemble.base.parse.InterviewSourceString(content=content, directory=the_directory, package="docassemble.playground" + str(current_user.id) + project_name(current_project), path="docassemble.playground" + str(current_user.id) + project_name(current_project) + ":" + active_file, testing=True)
        interview = interview_source.get_interview()
        ensure_ml_file_exists(interview, active_file, current_project)
        interview_status = docassemble.base.parse.InterviewStatus(current_info=current_info(yaml='docassemble.playground' + str(current_user.id) + project_name(current_project) + ':' + active_file, req=request, action=None, device_id=request.cookies.get('ds', None)))
        variables_html, vocab_list, vocab_dict = get_vars_in_use(interview, interview_status, debug_mode=False, current_project=current_project)
        return jsonify(success=True, variables_html=variables_html, vocab_list=vocab_list, current_project=current_project)
    return jsonify(success=False, reason=2)

def ensure_ml_file_exists(interview, yaml_file, current_project):
    if len(interview.mlfields) > 0:
        if hasattr(interview, 'ml_store'):
            parts = interview.ml_store.split(':')
            if parts[0] != 'docassemble.playground' + str(current_user.id) + current_project:
                return
            source_filename = re.sub(r'.*/', '', parts[1])
        else:
            source_filename = 'ml-' + re.sub(r'\.ya?ml$', '', yaml_file) + '.json'
        #logmessage("Source filename is " + source_filename)
        source_dir = SavedFile(current_user.id, fix=False, section='playgroundsources')
        source_directory = directory_for(source_dir, current_project)
        if current_project != 'default':
            source_filename = os.path.join(current_project, source_filename)
        if source_filename not in source_dir.list_of_files():
            #logmessage("Source filename does not exist yet")
            source_dir.fix()
            source_path = os.path.join(source_directory, source_filename)
            with open(source_path, 'a', encoding='utf-8'):
                os.utime(source_path, None)
            source_dir.finalize()

def assign_opacity(files):
    if len(files) == 1:
        files[0]['opacity'] = 1.0
    else:
        indexno = 0.0
        max_indexno = float(len(files) - 1)
        for file_dict in sorted(files, key=lambda x: x['modtime']):
            file_dict['opacity'] = round(0.2 + 0.8*(indexno/max_indexno), 2)
            indexno += 1.0

@app.route('/playground_run', methods=['GET', 'POST'])
@login_required
@roles_required(['developer', 'admin'])
def playground_page_run():
    if not app.config['ENABLE_PLAYGROUND']:
        return ('File not found', 404)
    setup_translation()
    current_project = get_current_project()
    the_file = secure_filename_spaces_ok(request.args.get('file'))
    if the_file:
        active_interview_string = 'docassemble.playground' + str(current_user.id) + project_name(current_project) + ':' + the_file
        the_url = url_for('index', reset=1, i=active_interview_string)
        key = 'da:runplayground:' + str(current_user.id)
        #logmessage("Setting key " + str(key) + " to " + str(the_url))
        pipe = r.pipeline()
        pipe.set(key, the_url)
        pipe.expire(key, 25)
        pipe.execute()
        return redirect(url_for('playground_page', file=the_file, project=current_project))
    return redirect(url_for('playground_page', project=current_project))

def get_list_of_projects(user_id):
    playground = SavedFile(user_id, fix=False, section='playground')
    return playground.list_of_dirs()

def rename_project(user_id, old_name, new_name):
    fix_package_folder()
    for sec in ('', 'sources', 'static', 'template', 'modules', 'packages'):
        area = SavedFile(user_id, fix=True, section='playground' + sec)
        if os.path.isdir(os.path.join(area.directory, old_name)):
            os.rename(os.path.join(area.directory, old_name), os.path.join(area.directory, new_name))
            area.finalize()

def create_project(user_id, new_name):
    fix_package_folder()
    for sec in ('', 'sources', 'static', 'template', 'modules', 'packages'):
        area = SavedFile(user_id, fix=True, section='playground' + sec)
        new_dir = os.path.join(area.directory, new_name)
        if not os.path.isdir(new_dir):
            os.makedirs(new_dir)
        path = os.path.join(new_dir, '.placeholder')
        with open(path, 'a', encoding='utf-8'):
            os.utime(path, None)
        area.finalize()

def delete_project(user_id, project_name):
    fix_package_folder()
    for sec in ('', 'sources', 'static', 'template', 'modules', 'packages'):
        area = SavedFile(user_id, fix=True, section='playground' + sec)
        area.delete_directory(project_name)
        area.finalize()

@app.route('/playgroundproject', methods=['GET', 'POST'])
@login_required
@roles_required(['developer', 'admin'])
def playground_project():
    setup_translation()
    use_gd = bool(app.config['USE_GOOGLE_DRIVE'] is True and get_gd_folder() is not None)
    use_od = bool(use_gd is False and app.config['USE_ONEDRIVE'] is True and get_od_folder() is not None)
    current_project = get_current_project()
    if request.args.get('rename'):
        form = RenameProject(request.form)
        mode = 'rename'
        description = word("You are renaming the project called %s.") % (current_project, )
        page_title = word("Rename project")
        if request.method == 'POST' and form.validate():
            if current_project == 'default':
                flash(word("You cannot rename the default Playground project"), 'error')
            else:
                rename_project(current_user.id, current_project, form.name.data)
                if use_gd:
                    try:
                        rename_gd_project(current_project, form.name.data)
                    except Exception as the_err:
                        logmessage("playground_project: unable to rename project on Google Drive.  " + str(the_err))
                elif use_od:
                    try:
                        rename_od_project(current_project, form.name.data)
                    except Exception as the_err:
                        try:
                            logmessage("playground_project: unable to rename project on OneDrive.  " + str(the_err))
                        except:
                            logmessage("playground_project: unable to rename project on OneDrive.")
                current_project = set_current_project(form.name.data)
                flash(word('Since you renamed a project, the server needs to restart in order to reload any modules.'), 'info')
                return redirect(url_for('restart_page', next=url_for('playground_project', project=current_project)))
    elif request.args.get('new'):
        form = NewProject(request.form)
        mode = 'new'
        description = word("Enter the name of the new project you want to create.")
        page_title = word("New project")
        if request.method == 'POST' and form.validate():
            if form.name.data == 'default' or form.name.data in get_list_of_projects(current_user.id):
                flash(word("The project name %s is not available.") % (form.name.data, ), "error")
            else:
                create_project(current_user.id, form.name.data)
                current_project = set_current_project(form.name.data)
                mode = 'standard'
    elif request.args.get('delete'):
        form = DeleteProject(request.form)
        mode = 'delete'
        description = word("WARNING!  If you press Delete, the contents of the %s project will be permanently deleted.") % (current_project, )
        page_title = word("Delete project")
        if request.method == 'POST' and form.validate():
            if current_project == 'default':
                flash(word("The default project cannot be deleted."), "error")
            else:
                if use_gd:
                    try:
                        trash_gd_project(current_project)
                    except Exception as the_err:
                        logmessage("playground_project: unable to delete project on Google Drive.  " + str(the_err))
                elif use_od:
                    try:
                        trash_od_project(current_project)
                    except Exception as the_err:
                        try:
                            logmessage("playground_project: unable to delete project on OneDrive.  " + str(the_err))
                        except:
                            logmessage("playground_project: unable to delete project on OneDrive.")
                delete_project(current_user.id, current_project)
                flash(word("The project %s was deleted.") % (current_project,), "success")
                current_project = set_current_project('default')
                return redirect(url_for('playground_project', project=current_project))
    else:
        form = None
        mode = 'standard'
        page_title = word("Projects")
        description = word("You can divide up your Playground into multiple separate areas, apart from your default Playground area.  Each Project has its own question files and Folders.")
    response = make_response(render_template('pages/manage_projects.html', version_warning=None, bodyclass='daadminbody', tab_title=word("Projects"), description=description, page_title=page_title, projects=get_list_of_projects(current_user.id), current_project=current_project, mode=mode, form=form), 200)
    response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
    return response

def set_current_project(new_name):
    key = 'da:playground:project:' + str(current_user.id)
    pipe = r.pipeline()
    pipe.set(key, new_name)
    pipe.expire(key, 2592000)
    pipe.execute()
    return new_name

def get_current_project():
    current_project = request.args.get('project', None)
    if current_project is not None:
        current_project = werkzeug.utils.secure_filename(current_project)
    key = 'da:playground:project:' + str(current_user.id)
    if current_project is None:
        current_project = r.get(key)
        if current_project is not None:
            current_project = current_project.decode()
    else:
        pipe = r.pipeline()
        pipe.set(key, current_project)
        pipe.expire(key, 2592000)
        pipe.execute()
    if current_project is None:
        return 'default'
    return current_project

def set_current_file(current_project, section, new_name):
    key = 'da:playground:project:' + str(current_user.id) + ':playground' + section + ':' + current_project
    pipe = r.pipeline()
    pipe.set(key, new_name)
    pipe.expire(key, 2592000)
    pipe.execute()
    return new_name

def get_current_file(current_project, section):
    key = 'da:playground:project:' + str(current_user.id) + ':playground' + section + ':' + current_project
    current_file = r.get(key)
    if current_file is None:
        return ''
    return current_file.decode()

def delete_current_file(current_project, section):
    key = 'da:playground:project:' + str(current_user.id) + ':playground' + section + ':' + current_project
    r.delete(key)

def set_variable_file(current_project, variable_file):
    key = 'da:playground:project:' + str(current_user.id) + ':' + current_project + ':variablefile'
    pipe = r.pipeline()
    pipe.set(key, variable_file)
    pipe.expire(key, 2592000)
    pipe.execute()
    return variable_file

def get_variable_file(current_project):
    key = 'da:playground:project:' + str(current_user.id) + ':' + current_project + ':variablefile'
    variable_file = r.get(key)
    if variable_file is not None:
        variable_file = variable_file.decode()
    return variable_file

def delete_variable_file(current_project):
    key = 'da:playground:project:' + str(current_user.id) + ':' + current_project + ':variablefile'
    r.delete(key)

@app.route('/playground', methods=['GET', 'POST'])
@login_required
@roles_required(['developer', 'admin'])
def playground_page():
    if not app.config['ENABLE_PLAYGROUND']:
        return ('File not found', 404)
    setup_translation()
    current_project = get_current_project()
    if 'ajax' in request.form and int(request.form['ajax']):
        is_ajax = True
        use_gd = False
        use_od = False
    else:
        is_ajax = False
        use_gd = bool(app.config['USE_GOOGLE_DRIVE'] is True and get_gd_folder() is not None)
        use_od = bool(use_gd is False and app.config['USE_ONEDRIVE'] is True and get_od_folder() is not None)
        if request.method == 'GET' and needs_to_change_password():
            return redirect(url_for('user.change_password', next=url_for('playground_page', project=current_project)))
    fileform = PlaygroundUploadForm(request.form)
    form = PlaygroundForm(request.form)
    interview = None
    the_file = secure_filename_spaces_ok(request.args.get('file', get_current_file(current_project, 'questions')))
    valid_form = None
    if request.method == 'POST':
        valid_form = form.validate()
    if request.method == 'GET':
        is_new = true_or_false(request.args.get('new', False))
        debug_mode = true_or_false(request.args.get('debug', False))
    else:
        debug_mode = False
        is_new = bool(not valid_form and form.status.data == 'new')
    if is_new:
        the_file = ''
    playground = SavedFile(current_user.id, fix=True, section='playground')
    the_directory = directory_for(playground, current_project)
    if current_project != 'default' and not os.path.isdir(the_directory):
        current_project = set_current_project('default')
        the_directory = directory_for(playground, current_project)
    if request.method == 'POST' and 'uploadfile' in request.files:
        the_files = request.files.getlist('uploadfile')
        if the_files:
            for up_file in the_files:
                try:
                    filename = secure_filename(up_file.filename)
                    extension, mimetype = get_ext_and_mimetype(filename)
                    if extension not in ('yml', 'yaml'):
                        flash(word("Sorry, only YAML files can be uploaded here.  To upload other types of files, use the Folders."), 'error')
                        return redirect(url_for('playground_page', project=current_project))
                    filename = re.sub(r'[^A-Za-z0-9\-\_\. ]+', '_', filename)
                    new_file = filename
                    filename = os.path.join(the_directory, filename)
                    up_file.save(filename)
                    try:
                        with open(filename, 'r', encoding='utf-8') as fp:
                            fp.read()
                    except:
                        os.remove(filename)
                        flash(word("There was a problem reading the YAML file you uploaded.  Are you sure it is a YAML file?  File was not saved."), 'error')
                        return redirect(url_for('playground_page', project=current_project))
                    playground.finalize()
                    r.incr('da:interviewsource:docassemble.playground' + str(current_user.id) + project_name(current_project) + ':' + new_file)
                    return redirect(url_for('playground_page', project=current_project, file=os.path.basename(filename)))
                except Exception as errMess:
                    flash("Error of type " + str(type(errMess)) + " processing upload: " + str(errMess), "error")
        return redirect(url_for('playground_page', project=current_project))
    if request.method == 'POST' and (form.submit.data or form.run.data or form.delete.data):
        if form.validate() and form.playground_name.data:
            the_file = secure_filename_spaces_ok(form.playground_name.data)
            #the_file = re.sub(r'[^A-Za-z0-9\_\-\. ]', '', the_file)
            if the_file != '':
                if not re.search(r'\.ya?ml$', the_file):
                    the_file = re.sub(r'\..*', '', the_file) + '.yml'
                filename = os.path.join(the_directory, the_file)
                if not os.path.isfile(filename):
                    with open(filename, 'a', encoding='utf-8'):
                        os.utime(filename, None)
            else:
                #flash(word('You need to type in a name for the interview'), 'error')
                is_new = True
        else:
            #flash(word('You need to type in a name for the interview'), 'error')
            is_new = True
    #the_file = re.sub(r'[^A-Za-z0-9\_\-\. ]', '', the_file)
    files = sorted([dict(name=f, modtime=os.path.getmtime(os.path.join(the_directory, f))) for f in os.listdir(the_directory) if os.path.isfile(os.path.join(the_directory, f)) and re.search(r'^[A-Za-z0-9].*[A-Za-z]$', f)], key=lambda x: x['name'])
    file_listing = [x['name'] for x in files]
    assign_opacity(files)
    if valid_form is False:
        content = form.playground_content.data
    else:
        content = ''
    if the_file and not is_new and the_file not in file_listing:
        the_file = ''
    is_default = False
    if request.method == 'GET' and not the_file and not is_new:
        current_file = get_current_file(current_project, 'questions')
        if current_file in files:
            the_file = current_file
        else:
            delete_current_file(current_project, 'questions')
            if len(files) > 0:
                the_file = sorted(files, key=lambda x: x['modtime'])[-1]['name']
            else:
                the_file = 'test.yml'
                is_default = True
                content = default_playground_yaml
    if the_file in file_listing:
        set_current_file(current_project, 'questions', the_file)
    active_file = the_file
    current_variable_file = get_variable_file(current_project)
    if current_variable_file is not None:
        if current_variable_file in file_listing:
            active_file = current_variable_file
        else:
            delete_variable_file(current_project)
    if the_file != '':
        filename = os.path.join(the_directory, the_file)
        if not os.path.isfile(filename):
            with open(filename, 'w', encoding='utf-8') as fp:
                fp.write(content)
            playground.finalize()
    console_messages = []
    if request.method == 'POST' and the_file != '' and form.validate():
        if form.delete.data:
            filename_to_del = os.path.join(the_directory, form.playground_name.data)
            if os.path.isfile(filename_to_del):
                os.remove(filename_to_del)
                flash(word('File deleted.'), 'info')
                r.delete('da:interviewsource:docassemble.playground' + str(current_user.id) + project_name(current_project) + ':' + the_file)
                if active_file != the_file:
                    r.incr('da:interviewsource:docassemble.playground' + str(current_user.id) + project_name(current_project) + ':' + active_file)
                cloud_trash(use_gd, use_od, 'questions', form.playground_name.data, current_project)
                playground.finalize()
                current_variable_file = get_variable_file(current_project)
                if current_variable_file == the_file or current_variable_file == form.playground_name.data:
                    delete_variable_file(current_project)
                return redirect(url_for('playground_page', project=current_project))
            else:
                flash(word('File not deleted.  There was an error.'), 'error')
        if (form.submit.data or form.run.data) and form.playground_content.data:
            if form.original_playground_name.data and form.original_playground_name.data != the_file:
                old_filename = os.path.join(the_directory, form.original_playground_name.data)
                if not is_ajax:
                    flash(word("Changed name of interview"), 'success')
                cloud_trash(use_gd, use_od, 'questions', form.original_playground_name.data, current_project)
                if os.path.isfile(old_filename):
                    os.remove(old_filename)
                    files = sorted([dict(name=f, modtime=os.path.getmtime(os.path.join(the_directory, f))) for f in os.listdir(the_directory) if os.path.isfile(os.path.join(the_directory, f)) and re.search(r'^[A-Za-z0-9].*[A-Za-z]$', f)], key=lambda x: x['name'])
                    file_listing = [x['name'] for x in files]
                    assign_opacity(files)
            the_time = formatted_current_time()
            should_save = True
            the_content = re.sub(r'\r\n', r'\n', form.playground_content.data)
            if os.path.isfile(filename):
                with open(filename, 'r', encoding='utf-8') as fp:
                    orig_content = fp.read()
                    if orig_content == the_content:
                        #logmessage("No need to save")
                        should_save = False
            if should_save:
                with open(filename, 'w', encoding='utf-8') as fp:
                    fp.write(the_content)
            if not form.submit.data and active_file != the_file:
                active_file = the_file
                set_variable_file(current_project, active_file)
            this_interview_string = 'docassemble.playground' + str(current_user.id) + project_name(current_project) + ':' + the_file
            active_interview_string = 'docassemble.playground' + str(current_user.id) + project_name(current_project) + ':' + active_file
            r.incr('da:interviewsource:' + this_interview_string)
            if the_file != active_file:
                r.incr('da:interviewsource:' + active_interview_string)
            playground.finalize()
            docassemble.base.interview_cache.clear_cache(this_interview_string)
            if active_interview_string != this_interview_string:
                docassemble.base.interview_cache.clear_cache(active_interview_string)
            if not form.submit.data:
                the_url = url_for('index', reset=1, i=this_interview_string)
                key = 'da:runplayground:' + str(current_user.id)
                #logmessage("Setting key " + str(key) + " to " + str(the_url))
                pipe = r.pipeline()
                pipe.set(key, the_url)
                pipe.expire(key, 12)
                pipe.execute()
            try:
                interview_source = docassemble.base.parse.interview_source_from_string(active_interview_string)
                interview_source.set_testing(True)
                interview = interview_source.get_interview()
                ensure_ml_file_exists(interview, active_file, current_project)
                interview_status = docassemble.base.parse.InterviewStatus(current_info=current_info(yaml='docassemble.playground' + str(current_user.id) + project_name(current_project) + ':' + active_file, req=request, action=None, device_id=request.cookies.get('ds', None)))
                variables_html, vocab_list, vocab_dict = get_vars_in_use(interview, interview_status, debug_mode=debug_mode, current_project=current_project)
                if form.submit.data:
                    flash_message = flash_as_html(word('Saved at') + ' ' + the_time + '.', 'success', is_ajax=is_ajax)
                else:
                    flash_message = flash_as_html(word('Saved at') + ' ' + the_time + '.  ' + word('Running in other tab.'), message_type='success', is_ajax=is_ajax)
                if interview.issue.get('mandatory_id', False):
                    console_messages.append(word("Note: it is a best practice to tag every mandatory block with an id."))
                if interview.issue.get('id_collision', False):
                    console_messages.append(word("Note: more than one block uses id") + " " + interview.issue['id_collision'])
            except DAError as foo:
                variables_html = None
                flash_message = flash_as_html(word('Saved at') + ' ' + the_time + '.  ' + word('Problem detected.'), message_type='error', is_ajax=is_ajax)
            if is_ajax:
                return jsonify(variables_html=variables_html, vocab_list=vocab_list, flash_message=flash_message, current_project=current_project, console_messages=console_messages, active_file=active_file, active_interview_url=url_for('index', i=active_interview_string))
        else:
            flash(word('Playground not saved.  There was an error.'), 'error')
    interview_path = None
    if the_file != '':
        with open(filename, 'r', encoding='utf-8') as fp:
            form.original_playground_name.data = the_file
            form.playground_name.data = the_file
            content = fp.read()
            #if not form.playground_content.data:
                #form.playground_content.data = content
    if active_file != '':
        is_fictitious = False
        interview_path = 'docassemble.playground' + str(current_user.id) + project_name(current_project) + ':' + active_file
        if is_default:
            interview_source = docassemble.base.parse.InterviewSourceString(content=content, directory=the_directory, package="docassemble.playground" + str(current_user.id) + project_name(current_project), path="docassemble.playground" + str(current_user.id) + project_name(current_project) + ":" + active_file, testing=True)
        else:
            interview_source = docassemble.base.parse.interview_source_from_string(interview_path)
            interview_source.set_testing(True)
    else:
        is_fictitious = True
        active_file = 'test.yml'
        if form.playground_content.data:
            content = re.sub(r'\r', '', form.playground_content.data)
            interview_source = docassemble.base.parse.InterviewSourceString(content=content, directory=the_directory, package="docassemble.playground" + str(current_user.id) + project_name(current_project), path="docassemble.playground" + str(current_user.id) + project_name(current_project) + ":" + active_file, testing=True)
        else:
            interview_source = docassemble.base.parse.InterviewSourceString(content='', directory=the_directory, package="docassemble.playground" + str(current_user.id) + project_name(current_project), path="docassemble.playground" + str(current_user.id) + project_name(current_project) + ":" + active_file, testing=True)
    interview = interview_source.get_interview()
    if hasattr(interview, 'mandatory_id_issue') and interview.mandatory_id_issue:
        console_messages.append(word("Note: it is a best practice to tag every mandatory block with an id."))
    interview_status = docassemble.base.parse.InterviewStatus(current_info=current_info(yaml='docassemble.playground' + str(current_user.id) + project_name(current_project) + ':' + active_file, req=request, action=None, device_id=request.cookies.get('ds', None)))
    variables_html, vocab_list, vocab_dict = get_vars_in_use(interview, interview_status, debug_mode=debug_mode, current_project=current_project)
    pulldown_files = [x['name'] for x in files]
    define_examples()
    if is_fictitious or is_new or is_default:
        new_active_file = word('(New file)')
        if new_active_file not in pulldown_files:
            pulldown_files.insert(0, new_active_file)
        if is_fictitious:
            active_file = new_active_file
    ajax = """
var exampleData;
var originalFileName = """ + json.dumps(the_file) + """;
var isNew = """ + json.dumps(is_new) + """;
var vocab = """ + json.dumps(vocab_list) + """;
var existingFiles = """ + json.dumps(file_listing) + """;
var currentProject = """ + json.dumps(current_project) + """;
var currentFile = """ + json.dumps(the_file) + """;
var attrs_showing = Object();
var daExpireSession = null;
var daNotificationContainer = """ + json.dumps(NOTIFICATION_CONTAINER) + """;
var daNotificationMessage = """ + json.dumps(NOTIFICATION_MESSAGE) + """;
Object.defineProperty(String.prototype, "daSprintf", {
  value: function () {
    var args = Array.from(arguments),
      i = 0;
    function defaultNumber(iValue) {
      return iValue != undefined && !isNaN(iValue) ? iValue : "0";
    }
    function defaultString(iValue) {
      return iValue == undefined ? "" : "" + iValue;
    }
    return this.replace(
      /%%|%([+\\-])?([^1-9])?(\\d+)?(\\.\\d+)?([deEfhHioQqs])/g,
      function (match, sign, filler, scale, precision, type) {
        var strOut, space, value;
        var asNumber = false;
        if (match == "%%") return "%";
        if (i >= args.length) return match;
        value = args[i];
        while (Array.isArray(value)) {
          args.splice(i, 1);
          for (var j = i; value.length > 0; j++)
            args.splice(j, 0, value.shift());
          value = args[i];
        }
        i++;
        if (filler == undefined) filler = " "; // default
        if (scale == undefined && !isNaN(filler)) {
          scale = filler;
          filler = " ";
        }
        if (sign == undefined) sign = "sqQ".indexOf(type) >= 0 ? "+" : "-"; // default
        if (scale == undefined) scale = 0; // default
        if (precision == undefined) precision = ".0"; // default
        scale = parseInt(scale);
        precision = parseInt(precision.substr(1));
        switch (type) {
          case "d":
          case "i":
            // decimal integer
            asNumber = true;
            strOut = parseInt(defaultNumber(value));
            if (precision > 0) strOut += "." + "0".repeat(precision);
            break;
          case "e":
          case "E":
            // float in exponential notation
            asNumber = true;
            strOut = parseFloat(defaultNumber(value));
            if (precision == 0) strOut = strOut.toExponential();
            else strOut = strOut.toExponential(precision);
            if (type == "E") strOut = strOut.replace("e", "E");
            break;
          case "f":
            // decimal float
            asNumber = true;
            strOut = parseFloat(defaultNumber(value));
            if (precision != 0) strOut = strOut.toFixed(precision);
            break;
          case "o":
          case "h":
          case "H":
            // Octal or Hexagesimal integer notation
            strOut =
              "\\\\" +
              (type == "o" ? "0" : type) +
              parseInt(defaultNumber(value)).toString(type == "o" ? 8 : 16);
            break;
          case "q":
            // single quoted string
            strOut = "'" + defaultString(value) + "'";
            break;
          case "Q":
            // double quoted string
            strOut = '"' + defaultString(value) + '"';
            break;
          default:
            // string
            strOut = defaultString(value);
            break;
        }
        if (typeof strOut != "string") strOut = "" + strOut;
        if ((space = strOut.length) < scale) {
          if (asNumber) {
            if (sign == "-") {
              if (strOut.indexOf("-") < 0)
                strOut = filler.repeat(scale - space) + strOut;
              else
                strOut =
                  "-" +
                  filler.repeat(scale - space) +
                  strOut.replace("-", "");
            } else {
              if (strOut.indexOf("-") < 0)
                strOut = "+" + filler.repeat(scale - space - 1) + strOut;
              else
                strOut =
                  "-" +
                  filler.repeat(scale - space) +
                  strOut.replace("-", "");
            }
          } else {
            if (sign == "-") strOut = filler.repeat(scale - space) + strOut;
            else strOut = strOut + filler.repeat(scale - space);
          }
        } else if (asNumber && sign == "+" && strOut.indexOf("-") < 0)
          strOut = "+" + strOut;
        return strOut;
      }
    );
  },
});
Object.defineProperty(window, "daSprintf", {
  value: function (str, ...rest) {
    if (typeof str == "string")
      return String.prototype.daSprintf.apply(str, rest);
    return "";
  },
});

function resetExpireSession(){
  if (daExpireSession != null){
    window.clearTimeout(daExpireSession);
  }
  daExpireSession = setTimeout(function(){
    alert(""" + json.dumps(word("Your browser session has expired and you have been signed out.  You will not be able to save your work.  Please log in again.")) + """);
  }, """ + str(999 * int(daconfig.get('session lifetime seconds', 43200))) + """);
}

""" + variables_js() + """

""" + search_js() + """

function activateExample(id, scroll){
  var info = exampleData[id];
  $("#da-example-source").html(info['html']);
  $("#da-example-source-before").html(info['before_html']);
  $("#da-example-source-after").html(info['after_html']);
  $("#da-example-image-link").attr("href", info['interview']);
  $("#da-example-image").attr("src", info['image']);
  if (info['documentation'] != null){
    $("#da-example-documentation-link").attr("href", info['documentation']);
    $("#da-example-documentation-link").removeClass("da-example-hidden");
    //$("#da-example-documentation-link").slideUp();
  }
  else{
    $("#da-example-documentation-link").addClass("da-example-hidden");
    //$("#da-example-documentation-link").slideDown();
  }
  $(".da-example-list").addClass("da-example-hidden");
  $(".da-example-link").removeClass("da-example-active");
  $(".da-example-link").removeClass("active");
  $(".da-example-link").each(function(){
    if ($(this).data("example") == id){
      $(this).addClass("da-example-active");
      $(this).addClass("active");
      $(this).parents(".da-example-list").removeClass("da-example-hidden");
      if (scroll){
        setTimeout(function(){
          //console.log($(this).parents("li").last()[0].offsetTop);
          //console.log($(this).parents("li").last().parent()[0].offsetTop);
          $(".da-example-active").parents("ul").last().scrollTop($(".da-example-active").parents("li").last()[0].offsetTop);
        }, 0);
      }
      //$(this).parents(".da-example-list").slideDown();
    }
  });
  $("#da-hide-full-example").addClass("dainvisible");
  if (info['has_context']){
    $("#da-show-full-example").removeClass("dainvisible");
  }
  else{
    $("#da-show-full-example").addClass("dainvisible");
  }
  $("#da-example-source-before").addClass("dainvisible");
  $("#da-example-source-after").addClass("dainvisible");
}

function daFetchVariableReportCallback(data){
  var translations = """ + json.dumps({'in mako': word("in mako"), 'mentioned in': word("mentioned in"), 'defined by': word("defined by")}) + """;
  var modal = $("#daVariablesReport .modal-body");
  if (modal.length == 0){
    console.log("No modal body on page");
    return;
  }
  if (!data.success){
    $(modal).html('<p>""" + word("Failed to load report") + """</p>');
    return;
  }
  var yaml_file = data.yaml_file;
  console.log(yaml_file)
  modal.empty();
  var accordion = $('<div>');
  accordion.addClass("accordion");
  accordion.attr("id", "varsreport");
  var n = data.items.length;
  for (var i = 0; i < n; ++i){
    var item = data.items[i];
    if (item.questions.length){
      var accordionItem = $('<div>');
      accordionItem.addClass("accordion-item");
      var accordionItemHeader = $('<h2>');
      accordionItemHeader.addClass("accordion-header");
      accordionItemHeader.attr("id", "accordionItemheader" + i);
      accordionItemHeader.html('<button class="accordion-button" type="button" data-bs-toggle="collapse" data-bs-target="#collapse' + i + '" aria-expanded="false" aria-controls="collapse' + i + '">' + item.name + '</button>');
      accordionItem.append(accordionItemHeader);
      var collapse = $("<div>");
      collapse.attr("id", "collapse" + i);
      collapse.attr("aria-labelledby", "accordionItemheader" + i);
      collapse.data("parent", "#varsreport");
      collapse.addClass("accordion-collapse");
      collapse.addClass("collapse");
      var accordionItemBody = $("<div>");
      accordionItemBody.addClass("accordion-body");
      var m = item.questions.length;
      for (var j = 0; j < m; j++){
        var h5 = $("<h5>");
        h5.html(item.questions[j].usage.map(x => translations[x]).join(','));
        var pre = $("<pre>");
        pre.html(item.questions[j].source_code);
        accordionItemBody.append(h5);
        accordionItemBody.append(pre);
        if (item.questions[j].yaml_file != yaml_file){
          var p = $("<p>");
          p.html(""" + json.dumps(word("from")) + """ + ' ' + item.questions[j].yaml_file);
          accordionItemBody.append(p);
        }
      }
      collapse.append(accordionItemBody);
      accordionItem.append(collapse);
      accordion.append(accordionItem);
    }
  }
  modal.append(accordion);
}

function daFetchVariableReport(){
  url = """ + json.dumps(url_for('variables_report', project=current_project)) + """ + "&file=" + currentFile;
  $("#daVariablesReport .modal-body").html('<p>""" + word("Loading . . .") + """</p>');
  $.ajax({
    type: "GET",
    url: url,
    success: daFetchVariableReportCallback,
    xhrFields: {
      withCredentials: true
    },
    error: function(xhr, status, error){
      $("#daVariablesReport .modal-body").html('<p>""" + word("Failed to load report") + """</p>');
    }
  });
}

function saveCallback(data){
  if (data.action && data.action == 'reload'){
    location.reload(true);
  }
  if ($("#daflash").length){
    $("#daflash").html(data.flash_message);
  }
  else{
    $("#damain").prepend(daSprintf(daNotificationContainer, data.flash_message));
  }
  if (data.vocab_list != null){
    vocab = data.vocab_list;
  }
  if (data.current_project != null){
    currentProject = data.current_project;
  }
  history.replaceState({}, "", """ + json.dumps(url_for('playground_page')) + """ + encodeURI('?project=' + currentProject + '&file=' + currentFile));
  $("#daVariables").val(data.active_file);
  $("#share-link").attr('href', data.active_interview_url);
  if (data.variables_html != null){
    $("#daplaygroundtable").html(data.variables_html);
    activateVariables();
    $("#form").trigger("reinitialize.areYouSure");
    var daPopoverTriggerList = [].slice.call(document.querySelectorAll('[data-bs-toggle="popover"]'));
    var daPopoverList = daPopoverTriggerList.map(function (daPopoverTriggerEl) {
      return new bootstrap.Popover(daPopoverTriggerEl, {trigger: "focus", html: true});
    });
  }
  daConsoleMessages = data.console_messages;
  daShowConsoleMessages();
}

function daShowConsoleMessages(){
  for (i=0; i < daConsoleMessages.length; ++i){
    console.log(daConsoleMessages[i]);
  }
}

function disableButtonsUntilCallback(){
  $("button.dasubmitbutton").prop('disabled', true);
  $("a.dasubmitbutton").addClass('dadisabled');
}

function enableButtons(){
  $(".dasubmitbutton").prop('disabled', false);
  $("a.dasubmitbutton").removeClass('dadisabled');
}

$( document ).ready(function() {
  variablesReady();
  searchReady();
  resetExpireSession();
  $("#playground_name").on('change', function(){
    var newFileName = $(this).val();
    if ((!isNew) && newFileName == currentFile){
      return;
    }
    for (var i = 0; i < existingFiles.length; i++){
      if (newFileName == existingFiles[i] || newFileName + '.yml' == existingFiles[i]){
        alert(""" + json.dumps(word("Warning: a file by that name already exists.  If you save, you will overwrite it.")) + """);
        return;
      }
    }
    return;
  });
  $("#daRun").click(function(event){
    if (originalFileName != $("#playground_name").val()){
      $("#form button[name='submit']").click();
      event.preventDefault();
      return false;
    }
    daCodeMirror.save();
    disableButtonsUntilCallback();
    $.ajax({
      type: "POST",
      url: """ + '"' + url_for('playground_page', project=current_project) + '"' + """,
      data: $("#form").serialize() + '&run=Save+and+Run&ajax=1',
      success: function(data){
        if (data.action && data.action == 'reload'){
          location.reload(true);
        }
        enableButtons();
        resetExpireSession();
        saveCallback(data);
      },
      dataType: 'json'
    });
    //event.preventDefault();
    return true;
  });
  var thisWindow = window;
  $("#daRunSyncGD").click(function(event){
    daCodeMirror.save();
    $("#form").trigger("checkform.areYouSure");
    if ($('#form').hasClass('dirty') && !confirm(""" + json.dumps(word("There are unsaved changes.  Are you sure you wish to leave this page?")) + """)){
      event.preventDefault();
      return false;
    }
    thisWindow.location.replace('""" + url_for('sync_with_google_drive', project=current_project, auto_next=url_for('playground_page_run', file=the_file, project=current_project)) + """');
    return true;
  });
  $("#daRunSyncOD").click(function(event){
    daCodeMirror.save();
    $("#form").trigger("checkform.areYouSure");
    if ($('#form').hasClass('dirty') && !confirm(""" + json.dumps(word("There are unsaved changes.  Are you sure you wish to leave this page?")) + """)){
      event.preventDefault();
      return false;
    }
    thisWindow.location.replace('""" + url_for('sync_with_onedrive', project=current_project, auto_next=url_for('playground_page_run', file=the_file, project=current_project)) + """');
  });
  $("#form button[name='submit']").click(function(event){
    daCodeMirror.save();
    if (isNew == "True" || originalFileName != $("#playground_name").val() || $("#playground_name").val().trim() == ""){
      return true;
    }
    disableButtonsUntilCallback();
    $.ajax({
      type: "POST",
      url: """ + '"' + url_for('playground_page', project=current_project) + '"' + """,
      data: $("#form").serialize() + '&submit=Save&ajax=1',
      success: function(data){
        if (data.action && data.action == 'reload'){
          location.reload(true);
        }
        enableButtons();
        resetExpireSession();
        saveCallback(data);
        setTimeout(function(){
          $("#daflash .alert-success").hide(300, function(){
            $(self).remove();
          });
        }, 3000);
      },
      dataType: 'json'
    });
    event.preventDefault();
    return false;
  });

  $(".da-example-link").on("click", function(){
    var id = $(this).data("example");
    activateExample(id, false);
  });

  $(".da-example-copy").on("click", function(event){
    if (daCodeMirror.somethingSelected()){
      daCodeMirror.replaceSelection("");
    }
    var id = $(".da-example-active").data("example");
    var curPos = daCodeMirror.getCursor();
    var notFound = 1;
    var insertLine = daCodeMirror.lastLine();
    daCodeMirror.eachLine(curPos.line, insertLine, function(line){
      if (notFound){
        if (line.text.substring(0, 3) == "---" || line.text.substring(0, 3) == "..."){
          insertLine = daCodeMirror.getLineNumber(line)
          //console.log("Found break at line number " + insertLine)
          notFound = 0;
        }
      }
    });
    if (notFound){
      daCodeMirror.setSelection({'line': insertLine, 'ch': null});
      daCodeMirror.replaceSelection("\\n---\\n" + exampleData[id]['source'] + "\\n", "around");
    }
    else{
      daCodeMirror.setSelection({'line': insertLine, 'ch': 0});
      daCodeMirror.replaceSelection("---\\n" + exampleData[id]['source'] + "\\n", "around");
    }
    daCodeMirror.focus();
    event.preventDefault();
    return false;
  });

  $(".da-example-heading").on("click", function(){
    var list = $(this).parent().children("ul").first();
    if (list != null){
      if (!list.hasClass("da-example-hidden")){
        return;
      }
      $(".da-example-list").addClass("da-example-hidden");
      //$(".da-example-list").slideUp();
      var new_link = $(this).parent().find("a.da-example-link").first();
      if (new_link.length){
        var id = new_link.data("example");
        activateExample(id, true);
      }
    }
  });

  activatePopovers();

  $("#da-show-full-example").on("click", function(){
    var id = $(".da-example-active").data("example");
    var info = exampleData[id];
    $(this).addClass("dainvisible");
    $("#da-hide-full-example").removeClass("dainvisible");
    $("#da-example-source-before").removeClass("dainvisible");
    $("#da-example-source-after").removeClass("dainvisible");
  });

  $("#da-hide-full-example").on("click", function(){
    var id = $(".da-example-active").data("example");
    var info = exampleData[id];
    $(this).addClass("dainvisible");
    $("#da-show-full-example").removeClass("dainvisible");
    $("#da-example-source-before").addClass("dainvisible");
    $("#da-example-source-after").addClass("dainvisible");
  });
  if ($("#playground_name").val().length > 0){
    daCodeMirror.focus();
  }
  else{
    $("#playground_name").focus()
  }
  activateVariables();
  updateRunLink();
  origPosition = daCodeMirror.getCursor();
  daShowConsoleMessages();
  if (currentFile != ''){
    history.replaceState({}, "", """ + json.dumps(url_for('playground_page')) + """ + encodeURI('?project=' + currentProject + '&file=' + currentFile));
  }
});
"""
    any_files = len(files) > 0
    cm_setup = """
    <script>
      var word_re = /[\w$]+/
      $( document ).ready(function(){
        CodeMirror.registerHelper("hint", "yaml", function(editor, options){
          var cur = editor.getCursor(), curLine = editor.getLine(cur.line);
          var end = cur.ch, start = end;
          while (start && word_re.test(curLine.charAt(start - 1))) --start;
          var curWord = start != end && curLine.slice(start, end);
          var list = [];
          if (curWord){
            var n = vocab.length;
            for (var i = 0; i < n; ++i){
              if (vocab[i].indexOf(curWord) == 0){
                list.push(vocab[i]);
              }
            }
          }
          return {list: list, from: CodeMirror.Pos(cur.line, start), to: CodeMirror.Pos(cur.line, end)};
        });""" + upload_js() + """
      });
    </script>"""
    if keymap:
        kbOpt = 'keyMap: "' + keymap + '", cursorBlinkRate: 0, '
        kbLoad = '<script src="' + url_for('static', filename="codemirror/keymap/" + keymap + ".js", v=da_version) + '"></script>\n    '
    else:
        kbOpt = ''
        kbLoad = ''
    page_title = word("Playground")
    if current_project != 'default':
        page_title += " / " + current_project
    response = make_response(render_template('pages/playground.html', projects=get_list_of_projects(current_user.id), current_project=current_project, version_warning=None, bodyclass='daadminbody', use_gd=use_gd, use_od=use_od, userid=current_user.id, page_title=Markup(page_title), tab_title=word("Playground"), extra_css=Markup('\n    <link href="' + url_for('static', filename='app/playgroundbundle.css', v=da_version) + '" rel="stylesheet">'), extra_js=Markup('\n    <script src="' + url_for('static', filename="app/playgroundbundle.js", v=da_version) + '"></script>\n    ' + kbLoad + cm_setup + '<script>\n      var daConsoleMessages = ' + json.dumps(console_messages) + ';\n      $("#daDelete").click(function(event){if(!confirm("' + word("Are you sure that you want to delete this playground file?") + '")){event.preventDefault();}});\n      daTextArea = document.getElementById("playground_content");\n      var daCodeMirror = CodeMirror.fromTextArea(daTextArea, {specialChars: /[\\u00a0\\u0000-\\u001f\\u007f-\\u009f\\u00ad\\u061c\\u200b-\\u200f\\u2028\\u2029\\ufeff]/, mode: "' + ('yamlmixed' if daconfig.get('test yamlmixed mode') else 'yamlmixed') + '", ' + kbOpt + 'tabSize: 2, tabindex: 70, autofocus: false, lineNumbers: true, matchBrackets: true, lineWrapping: ' + ('true' if daconfig.get('wrap lines in playground', True) else 'false') + '});\n      $(window).bind("beforeunload", function(){daCodeMirror.save(); $("#form").trigger("checkform.areYouSure");});\n      $("#form").areYouSure(' + json.dumps({'message': word("There are unsaved changes.  Are you sure you wish to leave this page?")}) + ');\n      $("#form").bind("submit", function(){daCodeMirror.save(); $("#form").trigger("reinitialize.areYouSure"); return true;});\n      daCodeMirror.setSize(null, null);\n      daCodeMirror.setOption("extraKeys", { Tab: function(cm) { var spaces = Array(cm.getOption("indentUnit") + 1).join(" "); cm.replaceSelection(spaces); }, "Ctrl-Space": "autocomplete", "F11": function(cm) { cm.setOption("fullScreen", !cm.getOption("fullScreen")); }, "Esc": function(cm) { if (cm.getOption("fullScreen")) cm.setOption("fullScreen", false); }});\n      daCodeMirror.setOption("coverGutterNextToScrollbar", true);\n' + indent_by(ajax, 6) + '\n      exampleData = JSON.parse(atob("' + pg_ex['encoded_data_dict'] + '"));\n      activateExample("' + str(pg_ex['pg_first_id'][0]) + '", false);\n    $("#my-form").trigger("reinitialize.areYouSure");\n      $("#daVariablesReport").on("shown.bs.modal", function () { daFetchVariableReport(); })\n    </script>'), form=form, fileform=fileform, files=sorted(files, key=lambda y: y['name'].lower()), any_files=any_files, pulldown_files=sorted(pulldown_files, key=lambda y: y.lower()), current_file=the_file, active_file=active_file, content=content, variables_html=Markup(variables_html), example_html=pg_ex['encoded_example_html'], interview_path=interview_path, is_new=str(is_new)), 200)
    response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
    return response

@app.errorhandler(404)
def page_not_found_error(the_error):
    return render_template('pages/404.html'), 404

@app.errorhandler(Exception)
def server_error(the_error):
    setup_translation()
    if hasattr(the_error, 'interview') and the_error.interview.debug and hasattr(the_error, 'interview_status'):
        the_history = get_history(the_error.interview, the_error.interview_status)
    else:
        the_history = None
    the_vars = None
    if isinstance(the_error, DAError):
        errmess = str(the_error)
        the_trace = None
        logmessage(errmess)
    elif isinstance(the_error, TemplateError):
        errmess = str(the_error)
        if hasattr(the_error, 'name') and the_error.name is not None:
            errmess += "\nName: " + str(the_error.name)
        if hasattr(the_error, 'filename') and the_error.filename is not None:
            errmess += "\nFilename: " + str(the_error.filename)
        if hasattr(the_error, 'docx_context'):
            errmess += "\n\nContext:\n" + "\n".join(map(lambda x: "  " + x, the_error.docx_context))
        the_trace = traceback.format_exc()
        try:
            logmessage(errmess)
        except:
            logmessage("Could not log the error message")
    else:
        try:
            errmess = str(type(the_error).__name__) + ": " + str(the_error)
        except:
            errmess = str(type(the_error).__name__)
        if hasattr(the_error, 'traceback'):
            the_trace = the_error.traceback
        else:
            the_trace = traceback.format_exc()
        if hasattr(docassemble.base.functions.this_thread, 'misc') and 'current_field' in docassemble.base.functions.this_thread.misc:
            errmess += "\nIn field index number " + str(docassemble.base.functions.this_thread.misc['current_field'])
        if hasattr(the_error, 'da_line_with_error'):
            errmess += "\nIn line: " + str(the_error.da_line_with_error)

        logmessage(the_trace)
    if isinstance(the_error, DAError):
        error_code = the_error.error_code
    elif isinstance(the_error, werkzeug.exceptions.HTTPException):
        error_code = the_error.code
    else:
        error_code = 501
    if hasattr(the_error, 'user_dict'):
        the_vars = the_error.user_dict
    if hasattr(the_error, 'interview'):
        special_error_markdown = the_error.interview.consolidated_metadata.get('error help', None)
        if isinstance(special_error_markdown, dict):
            language = docassemble.base.functions.get_language()
            if language in special_error_markdown:
                special_error_markdown = special_error_markdown[language]
            elif '*' in special_error_markdown:
                special_error_markdown = special_error_markdown['*']
            elif DEFAULT_LANGUAGE in special_error_markdown:
                special_error_markdown = special_error_markdown[DEFAULT_LANGUAGE]
            else:
                special_error_markdown = None
    else:
        special_error_markdown = None
    if special_error_markdown is None:
        special_error_markdown = daconfig.get('error help', None)
    if special_error_markdown is not None:
        special_error_html = docassemble.base.util.markdown_to_html(special_error_markdown)
    else:
        special_error_html = None
    flask_logtext = []
    if os.path.exists(LOGFILE):
        with open(LOGFILE, encoding='utf-8') as the_file:
            for line in the_file:
                if re.match('Exception', line):
                    flask_logtext = []
                flask_logtext.append(line)
    orig_errmess = errmess
    errmess = noquote(errmess)
    if re.search(r'\n', errmess):
        errmess = '<pre>' + errmess + '</pre>'
    else:
        errmess = '<blockquote class="blockquote">' + errmess + '</blockquote>'
    script = """
    <script>
      var daGlobalEval = eval;
      var daMessageLog = JSON.parse(atob(""" + json.dumps(safeid(json.dumps(docassemble.base.functions.get_message_log()))) + """));
      function flash(message, priority){
        if (priority == null){
          priority = 'info'
        }
        if (!$("#daflash").length){
          $(daTargetDiv).append('<div class="datopcenter col-sm-7 col-md-6 col-lg-5" id="daflash"></div>');
        }
        $("#daflash").append('<div class="da-alert alert alert-' + priority + ' daalert-interlocutory alert-dismissible fade show" role="alert">' + message + '<button type="button" class="btn-close" data-bs-dismiss="alert" aria-label="Close"><\/button><\/div>');
        if (priority == 'success'){
          setTimeout(function(){
            $("#daflash .alert-success").hide(300, function(){
              $(self).remove();
            });
          }, 3000);
        }
      }
      var da_flash = flash;
      function daShowNotifications(){
        var n = daMessageLog.length;
        for (var i = 0; i < n; i++){
          var message = daMessageLog[i];
          if (message.priority == 'console'){
            console.log(message.message);
          }
          else if (message.priority == 'javascript'){
            daGlobalEval(message.message);
          }
          else if (message.priority == 'success' || message.priority == 'warning' || message.priority == 'danger' || message.priority == 'secondary' || message.priority == 'info' || message.priority == 'secondary' || message.priority == 'dark' || message.priority == 'light' || message.priority == 'primary'){
            da_flash(message.message, message.priority);
          }
          else{
            da_flash(message.message, 'info');
          }
        }
      }
      $( document ).ready(function() {
        $("#da-retry").on('click', function(e){
          location.reload();
          e.preventDefault();
          return false;
        });
        daShowNotifications();
      });
    </script>"""
    error_notification(the_error, message=errmess, history=the_history, trace=the_trace, the_request=request, the_vars=the_vars)
    if request.path.endswith('/interview') and 'in error' not in session and docassemble.base.functions.this_thread.interview is not None and 'error action' in docassemble.base.functions.this_thread.interview.consolidated_metadata and docassemble.base.functions.interview_path() is not None:
        session['in error'] = True
        #session['action'] = docassemble.base.functions.myb64quote(json.dumps({'action': docassemble.base.functions.this_thread.interview.consolidated_metadata['error action'], 'arguments': dict(error_message=orig_errmess)}))
        return index(action_argument={'action': docassemble.base.functions.this_thread.interview.consolidated_metadata['error action'], 'arguments': dict(error_message=orig_errmess)}, refer=['error'])
    show_debug = not bool((not DEBUG) and isinstance(the_error, DAError))
    if int(int(error_code)/100) == 4:
        show_debug = False
    if error_code == 404:
        the_template = 'pages/404.html'
    else:
        the_template = 'pages/501.html'
    try:
        yaml_filename = docassemble.base.functions.interview_path()
    except:
        yaml_filename = None
    show_retry = request.path.endswith('/interview')
    return render_template(the_template, verbose=daconfig.get('verbose error messages', True), version_warning=None, tab_title=word("Error"), page_title=word("Error"), error=errmess, historytext=str(the_history), logtext=str(the_trace), extra_js=Markup(script), special_error=special_error_html, show_debug=show_debug, yaml_filename=yaml_filename, show_retry=show_retry), error_code

@app.route('/bundle.css', methods=['GET'])
def css_bundle():
    base_path = pkg_resources.resource_filename(pkg_resources.Requirement.parse('docassemble.webapp'), os.path.join('docassemble', 'webapp', 'static'))
    output = ''
    for parts in [['bootstrap-fileinput', 'css', 'fileinput.min.css'], ['labelauty', 'source', 'jquery-labelauty.min.css'], ['bootstrap-combobox', 'css', 'bootstrap-combobox.min.css'], ['bootstrap-slider', 'dist', 'css', 'bootstrap-slider.min.css'], ['app', 'app.min.css']]:
        with open(os.path.join(base_path, *parts), encoding='utf-8') as fp:
            output += fp.read()
        output += "\n"
    return Response(output, mimetype='text/css')

@app.route('/playgroundbundle.css', methods=['GET'])
def playground_css_bundle():
    base_path = pkg_resources.resource_filename(pkg_resources.Requirement.parse('docassemble.webapp'), os.path.join('docassemble', 'webapp', 'static'))
    output = ''
    for parts in [['codemirror', 'lib', 'codemirror.css'], ['codemirror', 'addon', 'search', 'matchesonscrollbar.css'], ['codemirror', 'addon', 'display', 'fullscreen.css'], ['codemirror', 'addon', 'scroll', 'simplescrollbars.css'], ['codemirror', 'addon', 'hint', 'show-hint.css'], ['app', 'pygments.min.css'], ['bootstrap-fileinput', 'css', 'fileinput.min.css']]:
        with open(os.path.join(base_path, *parts), encoding='utf-8') as fp:
            output += fp.read()
        output += "\n"
    return Response(output, mimetype='text/css')

@app.route('/bundle.js', methods=['GET'])
def js_bundle():
    base_path = pkg_resources.resource_filename(pkg_resources.Requirement.parse('docassemble.webapp'), os.path.join('docassemble', 'webapp', 'static'))
    output = ''
    for parts in [['app', 'jquery.min.js'], ['app', 'jquery.validate.min.js'], ['app', 'additional-methods.min.js'], ['app', 'jquery.visible.min.js'], ['bootstrap', 'js', 'bootstrap.bundle.min.js'], ['bootstrap-slider', 'dist', 'bootstrap-slider.min.js'], ['bootstrap-fileinput', 'js', 'plugins', 'piexif.min.js'], ['bootstrap-fileinput', 'js', 'fileinput.min.js'], ['bootstrap-fileinput', 'themes', 'fas', 'theme.min.js'], ['app', 'app.min.js'], ['app', 'socket.io.min.js'], ['labelauty', 'source', 'jquery-labelauty.min.js'], ['bootstrap-combobox', 'js', 'bootstrap-combobox.min.js']]:
        with open(os.path.join(base_path, *parts), encoding='utf-8') as fp:
            output += fp.read()
        output += "\n"
    return Response(output, mimetype='application/javascript')

@app.route('/playgroundbundle.js', methods=['GET'])
def playground_js_bundle():
    base_path = pkg_resources.resource_filename(pkg_resources.Requirement.parse('docassemble.webapp'), os.path.join('docassemble', 'webapp', 'static'))
    output = ''
    for parts in [['areyousure', 'jquery.are-you-sure.js'], ['codemirror', 'lib', 'codemirror.js'], ['codemirror', 'addon', 'search', 'searchcursor.js'], ['codemirror', 'addon', 'scroll', 'annotatescrollbar.js'], ['codemirror', 'addon', 'search', 'matchesonscrollbar.js'], ['codemirror', 'addon', 'display', 'fullscreen.js'], ['codemirror', 'addon', 'edit', 'matchbrackets.js'], ['codemirror', 'addon', 'hint', 'show-hint.js'], ['codemirror', 'mode', 'yaml', 'yaml.js'], ['codemirror', 'mode', 'python', 'python.js'], ['yamlmixed', 'yamlmixed.js'], ['codemirror', 'mode', 'markdown', 'markdown.js'], ['bootstrap-fileinput', 'js', 'plugins', 'piexif.min.js'], ['bootstrap-fileinput', 'js', 'fileinput.min.js'], ['bootstrap-fileinput', 'themes', 'fas', 'theme.min.js']]:
        with open(os.path.join(base_path, *parts), encoding='utf-8') as fp:
            output += fp.read()
        output += "\n"
    return Response(output, mimetype='application/javascript')

@app.route('/adminbundle.js', methods=['GET'])
def js_admin_bundle():
    base_path = pkg_resources.resource_filename(pkg_resources.Requirement.parse('docassemble.webapp'), os.path.join('docassemble', 'webapp', 'static'))
    output = ''
    for parts in [['app', 'jquery.min.js'], ['bootstrap', 'js', 'bootstrap.bundle.min.js']]:
        with open(os.path.join(base_path, *parts), encoding='utf-8') as fp:
            output += fp.read()
        output += "\n"
    return Response(output, mimetype='application/javascript')

@app.route('/bundlewrapjquery.js', methods=['GET'])
def js_bundle_wrap():
    base_path = pkg_resources.resource_filename(pkg_resources.Requirement.parse('docassemble.webapp'), os.path.join('docassemble', 'webapp', 'static'))
    output = '(function($) {'
    for parts in [['app', 'jquery.validate.min.js'], ['app', 'additional-methods.min.js'], ['app', 'jquery.visible.js'], ['bootstrap', 'js', 'bootstrap.bundle.min.js'], ['bootstrap-slider', 'dist', 'bootstrap-slider.min.js'], ['bootstrap-fileinput', 'js', 'plugins', 'piexif.min.js'], ['bootstrap-fileinput', 'js', 'fileinput.min.js'], ['bootstrap-fileinput', 'themes', 'fas', 'theme.min.js'], ['app', 'app.min.js'], ['app', 'socket.io.min.js'], ['labelauty', 'source', 'jquery-labelauty.min.js'], ['bootstrap-combobox', 'js', 'bootstrap-combobox.min.js']]:
        with open(os.path.join(base_path, *parts), encoding='utf-8') as fp:
            output += fp.read()
        output += "\n"
    output += '})(jQuery);'
    return Response(output, mimetype='application/javascript')

@app.route('/bundlenojquery.js', methods=['GET'])
def js_bundle_no_query():
    base_path = pkg_resources.resource_filename(pkg_resources.Requirement.parse('docassemble.webapp'), os.path.join('docassemble', 'webapp', 'static'))
    output = ''
    for parts in [['app', 'jquery.validate.min.js'], ['app', 'additional-methods.min.js'], ['app', 'jquery.visible.min.js'], ['bootstrap', 'js', 'bootstrap.bundle.min.js'], ['bootstrap-slider', 'dist', 'bootstrap-slider.min.js'], ['bootstrap-fileinput', 'js', 'plugins', 'piexif.min.js'], ['bootstrap-fileinput', 'js', 'fileinput.min.js'], ['bootstrap-fileinput', 'themes', 'fas', 'theme.min.js'], ['app', 'app.min.js'], ['app', 'socket.io.min.js'], ['labelauty', 'source', 'jquery-labelauty.min.js'], ['bootstrap-combobox', 'js', 'bootstrap-combobox.min.js']]:
        with open(os.path.join(base_path, *parts), encoding='utf-8') as fp:
            output += fp.read()
        output += "\n"
    output += ''
    return Response(output, mimetype='application/javascript')

@app.route('/packagestatic/<package>/<path:filename>', methods=['GET'])
def package_static(package, filename):
    try:
        attach = int(request.args.get('attachment', 0))
    except:
        attach = 0
    if '../' in filename:
        return ('File not found', 404)
    if package == 'fonts':
        return redirect(url_for('static', filename='bootstrap/fonts/' + filename, v=da_version))
    try:
        filename = re.sub(r'^\.+', '', filename)
        filename = re.sub(r'\/\.+', '\/', filename)
        the_file = docassemble.base.functions.package_data_filename(str(package) + ':data/static/' + str(filename))
    except:
        return ('File not found', 404)
    if the_file is None:
        return ('File not found', 404)
    if not os.path.isfile(the_file):
        return ('File not found', 404)
    extension, mimetype = get_ext_and_mimetype(the_file)
    response = send_file(the_file, mimetype=str(mimetype))
    if attach:
        filename = os.path.basename(filename)
        response.headers['Content-Disposition'] = 'attachment; filename=' + json.dumps(filename)
    return response

@app.route('/logfile/<filename>', methods=['GET'])
@login_required
@roles_required(['admin', 'developer'])
def logfile(filename):
    if LOGSERVER is None:
        the_file = os.path.join(LOG_DIRECTORY, filename)
        if not os.path.isfile(the_file):
            return ('File not found', 404)
    else:
        h = httplib2.Http()
        resp, content = h.request("http://" + LOGSERVER + ':8082', "GET")
        try:
            the_file, headers = urlretrieve("http://" + LOGSERVER + ':8082/' + urllibquote(filename))
        except:
            return ('File not found', 404)
    response = send_file(the_file, as_attachment=True, mimetype='text/plain', attachment_filename=filename, cache_timeout=0)
    response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
    return response

@app.route('/logs', methods=['GET', 'POST'])
@login_required
@roles_required(['admin', 'developer'])
def logs():
    setup_translation()
    form = LogForm(request.form)
    use_zip = true_or_false(request.args.get('zip', None))
    if LOGSERVER is None and use_zip:
        timezone = get_default_timezone()
        zip_archive = tempfile.NamedTemporaryFile(mode="wb", prefix="datemp", suffix=".zip", delete=False)
        zf = zipfile.ZipFile(zip_archive, mode='w')
        for f in os.listdir(LOG_DIRECTORY):
            zip_path = os.path.join(LOG_DIRECTORY, f)
            if f.startswith('.') or not os.path.isfile(zip_path):
                continue
            info = zipfile.ZipInfo(f)
            info.compress_type = zipfile.ZIP_DEFLATED
            info.external_attr = 0o644 << 16
            info.date_time = datetime.datetime.utcfromtimestamp(os.path.getmtime(zip_path)).replace(tzinfo=pytz.utc).astimezone(pytz.timezone(timezone)).timetuple()
            with open(zip_path, 'rb') as fp:
                zf.writestr(info, fp.read())
        zf.close()
        zip_file_name = re.sub(r'[^A-Za-z0-9_]+', '', app.config['APP_NAME']) + '_logs.zip'
        response = send_file(zip_archive.name, mimetype='application/zip', as_attachment=True, attachment_filename=zip_file_name)
        response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
        return response
    the_file = request.args.get('file', None)
    if the_file is not None:
        the_file = secure_filename_spaces_ok(the_file)
    default_filter_string = request.args.get('q', '')
    if request.method == 'POST' and form.file_name.data:
        the_file = form.file_name.data
    if the_file is not None and (the_file.startswith('.') or the_file.startswith('/') or the_file == ''):
        the_file = None
    if the_file is not None:
        the_file = secure_filename_spaces_ok(the_file)
    total_bytes = 0
    if LOGSERVER is None:
        call_sync()
        files = []
        for f in os.listdir(LOG_DIRECTORY):
            path = os.path.join(LOG_DIRECTORY, f)
            if not os.path.isfile(path):
                continue
            files.append(f)
            total_bytes += os.path.getsize(path)
        files = sorted(files)
        total_bytes = humanize.naturalsize(total_bytes)
        if the_file is None and len(files):
            if 'docassemble.log' in files:
                the_file = 'docassemble.log'
            else:
                the_file = files[0]
        if the_file is not None:
            filename = os.path.join(LOG_DIRECTORY, the_file)
    else:
        h = httplib2.Http()
        resp, content = h.request("http://" + LOGSERVER + ':8082', "GET")
        if int(resp['status']) >= 200 and int(resp['status']) < 300:
            files = [f for f in content.decode().split("\n") if f != '' and f is not None]
        else:
            return ('File not found', 404)
        if len(files) > 0:
            if the_file is None:
                the_file = files[0]
            filename, headers = urlretrieve("http://" + LOGSERVER + ':8082/' + urllibquote(the_file))
    if len(files) > 0 and not os.path.isfile(filename):
        flash(word("The file you requested does not exist."), 'error')
        if len(files) > 0:
            the_file = files[0]
            filename = os.path.join(LOG_DIRECTORY, files[0])
    if len(files) > 0:
        if request.method == 'POST' and form.submit.data and form.filter_string.data:
            default_filter_string = form.filter_string.data
        try:
            reg_exp = re.compile(default_filter_string)
        except:
            flash(word("The regular expression you provided could not be parsed."), 'error')
            default_filter_string = ''
        if default_filter_string == '':
            try:
                lines = tailer.tail(open(filename, encoding='utf-8'), 30)
            except:
                lines = [word('Unable to read log file; please download.')]
        else:
            temp_file = tempfile.NamedTemporaryFile(mode='a+', encoding='utf-8')
            with open(filename, 'r', encoding='utf-8') as fp:
                for line in fp:
                    if reg_exp.search(line):
                        temp_file.write(line)
            temp_file.seek(0)
            try:
                lines = tailer.tail(temp_file, 30)
            except:
                lines = [word('Unable to read log file; please download.')]
            temp_file.close()
        content = "\n".join(map(lambda x: x, lines))
    else:
        content = "No log files available"
    show_download_all = bool(LOGSERVER is None)
    response = make_response(render_template('pages/logs.html', version_warning=version_warning, bodyclass='daadminbody', tab_title=word("Logs"), page_title=word("Logs"), form=form, files=files, current_file=the_file, content=content, default_filter_string=default_filter_string, show_download_all=show_download_all, total_bytes=total_bytes), 200)
    response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
    return response

@app.route('/reqdev', methods=['GET', 'POST'])
@login_required
def request_developer():
    setup_translation()
    if not app.config['ENABLE_PLAYGROUND']:
        return ('File not found', 404)
    form = RequestDeveloperForm(request.form)
    recipients = []
    if request.method == 'POST':
        for user in db.session.execute(select(UserModel.id, UserModel.email).join(UserRoles, UserModel.id == UserRoles.user_id).join(Role, UserRoles.role_id == Role.id).where(and_(UserModel.active == True, Role.name == 'admin'))):
            if user.email not in recipients:
                recipients.append(user.email)
        body = "User " + str(current_user.email) + " (" + str(current_user.id) + ") has requested developer privileges.\n\n"
        if form.reason.data:
            body += "Reason given: " + str(form.reason.data) + "\n\n"
        body += "Go to " + url_for('edit_user_profile_page', id=current_user.id, _external=True) + " to change the user's privileges."
        msg = Message("Request for developer account from " + str(current_user.email), recipients=recipients, body=body)
        if not len(recipients):
            flash(word('No administrators could be found.'), 'error')
        else:
            try:
                da_send_mail(msg)
                flash(word('Your request was submitted.'), 'success')
            except:
                flash(word('We were unable to submit your request.'), 'error')
        return redirect(url_for('user.profile'))
    return render_template('users/request_developer.html', version_warning=None, bodyclass='daadminbody', tab_title=word("Developer Access"), page_title=word("Developer Access"), form=form)

def docx_variable_fix(variable):
    variable = re.sub(r'\\', '', variable)
    variable = re.sub(r'^([A-Za-z\_][A-Za-z\_0-9]*).*', r'\1', variable)
    return variable

def sanitize(default):
    default = re.sub(r'\n?\r\n?', "\n", str(default))
    if re.search(r'[\#\!\?\:\n\r\"\'\[\]\{\}]+', default):
        return "|\n" + docassemble.base.functions.indent(default, by=10)
    return default

def read_fields(filename, orig_file_name, input_format, output_format):
    if output_format == 'yaml':
        if input_format == 'pdf':
            fields = docassemble.base.pdftk.read_fields(filename)
            fields_seen = set()
            if fields is None:
                raise Exception(word("Error: no fields could be found in the file"))
            fields_output = "---\nquestion: " + word("Here is your document.") + "\nevent: " + 'some_event' + "\nattachment:" + "\n  - name: " + os.path.splitext(orig_file_name)[0] + "\n    filename: " + os.path.splitext(orig_file_name)[0] + "\n    pdf template file: " + re.sub(r'[^A-Za-z0-9\-\_\. ]+', '_', orig_file_name) + "\n    fields:\n"
            for field, default, pageno, rect, field_type, export_value in fields:
                if field not in fields_seen:
                    fields_output += '      - "' + str(field) + '": ' + sanitize(default) + "\n"
                    fields_seen.add(field)
            fields_output += "---"
            return fields_output
        if input_format == 'docx' or input_format == 'markdown':
            if input_format == 'docx':
                result_file = word_to_markdown(filename, 'docx')
                if result_file is None:
                    raise Exception(word("Error: no fields could be found in the file"))
                with open(result_file.name, 'r', encoding='utf-8') as fp:
                    result = fp.read()
            elif input_format == 'markdown':
                with open(filename, 'r', encoding='utf-8') as fp:
                    result = fp.read()
            fields = set()
            for variable in re.findall(r'{{[pr] \s*([^\}\s]+)\s*}}', result):
                fields.add(docx_variable_fix(variable))
            for variable in re.findall(r'{{\s*([^\}\s]+)\s*}}', result):
                fields.add(docx_variable_fix(variable))
            for variable in re.findall(r'{%[a-z]* for [A-Za-z\_][A-Za-z0-9\_]* in *([^\} ]+) *%}', result):
                fields.add(docx_variable_fix(variable))
            if len(fields) == 0:
                raise Exception(word("Error: no fields could be found in the file"))
            fields_output = "---\nquestion: " + word("Here is your document.") + "\nevent: " + 'some_event' + "\nattachment:" + "\n  - name: " + os.path.splitext(orig_file_name)[0] + "\n    filename: " + os.path.splitext(orig_file_name)[0] + "\n    docx template file: " + re.sub(r'[^A-Za-z0-9\-\_\. ]+', '_', orig_file_name) + "\n    fields:\n"
            for field in fields:
                fields_output += '      "' + field + '": ' + "Something\n"
            fields_output += "---"
            return fields_output
    if output_format == 'json':
        if input_format == 'pdf':
            default_text = word("something")
            output = dict(fields=[], default_values={}, types={}, locations={}, export_values={})
            fields = docassemble.base.pdftk.read_fields(filename)
            if fields is not None:
                fields_seen = set()
                for field, default, pageno, rect, field_type, export_value in fields:
                    real_default = str(default)
                    if real_default == default_text:
                        real_default = ''
                    if field not in fields_seen:
                        output['fields'].append(str(field))
                        output['default_values'][field] = real_default
                        output['types'][field] = re.sub(r"'", r'', str(field_type))
                        output['locations'][field] = dict(page=int(pageno), box=rect)
                        output['export_values'][field] = export_value
            return json.dumps(output, sort_keys=True, indent=2)
        if input_format == 'docx' or input_format == 'markdown':
            if input_format == 'docx':
                result_file = word_to_markdown(filename, 'docx')
                if result_file is None:
                    return json.dumps(dict(fields=[]), indent=2)
                with open(result_file.name, 'r', encoding='utf-8') as fp:
                    result = fp.read()
            elif input_format == 'markdown':
                with open(filename, 'r', encoding='utf-8') as fp:
                    result = fp.read()
            fields = set()
            for variable in re.findall(r'{{ *([^\} ]+) *}}', result):
                fields.add(docx_variable_fix(variable))
            for variable in re.findall(r'{%[a-z]* for [A-Za-z\_][A-Za-z0-9\_]* in *([^\} ]+) *%}', result):
                fields.add(docx_variable_fix(variable))
            return json.dumps(dict(fields=list(fields)), sort_keys=True, indent=2)

@app.route('/utilities', methods=['GET', 'POST'])
@login_required
@roles_required(['admin', 'developer'])
def utilities():
    setup_translation()
    form = Utilities(request.form)
    fields_output = None
    word_box = None
    uses_null = False
    file_type = None
    if request.method == 'GET' and needs_to_change_password():
        return redirect(url_for('user.change_password', next=url_for('utilities')))
    if request.method == 'POST':
        if 'language' in request.form:
            language = request.form['language']
            result = {}
            result[language] = {}
            existing = docassemble.base.functions.word_collection.get(language, {})
            if 'google' in daconfig and 'api key' in daconfig['google'] and daconfig['google']['api key']:
                try:
                    service = googleapiclient.discovery.build('translate', 'v2',
                                    developerKey=daconfig['google']['api key'])
                    use_google_translate = True
                except:
                    logmessage("utilities: attempt to call Google Translate failed")
                    use_google_translate = False
            else:
                use_google_translate = False
            words_to_translate = []
            for the_word in base_words:
                if the_word in existing and existing[the_word] is not None:
                    result[language][the_word] = existing[the_word]
                    continue
                words_to_translate.append(the_word)
            chunk_limit = daconfig.get('google translate words at a time', 20)
            chunks = []
            interim_list = []
            while len(words_to_translate):
                the_word = words_to_translate.pop(0)
                interim_list.append(the_word)
                if len(interim_list) >= chunk_limit:
                    chunks.append(interim_list)
                    interim_list = []
            if len(interim_list) > 0:
                chunks.append(interim_list)
            for chunk in chunks:
                if use_google_translate:
                    try:
                        resp = service.translations().list(
                            source='en',
                            target=language,
                            q=chunk
                        ).execute()
                    except Exception as errstr:
                        logmessage("utilities: translation failed: " + str(errstr))
                        resp = None
                    if isinstance(resp, dict) and 'translations' in resp and isinstance(resp['translations'], list) and len(resp['translations']) == len(chunk):
                        for index in range(len(chunk)):
                            if isinstance(resp['translations'][index], dict) and 'translatedText' in resp['translations'][index]:
                                result[language][chunk[index]] = re.sub(r'&#39;', r"'", str(resp['translations'][index]['translatedText']))
                            else:
                                result[language][chunk[index]] = 'XYZNULLXYZ'
                                uses_null = True
                    else:
                        for the_word in chunk:
                            result[language][the_word] = 'XYZNULLXYZ'
                        uses_null = True
                else:
                    for the_word in chunk:
                        result[language][the_word] = 'XYZNULLXYZ'
                    uses_null = True
            if form.systemfiletype.data == 'YAML':
                word_box = ruamel.yaml.safe_dump(result, default_flow_style=False, default_style = '"', allow_unicode=True, width=1000)
                word_box = re.sub(r'"XYZNULLXYZ"', r'null', word_box)
            elif form.systemfiletype.data == 'XLSX':
                temp_file = tempfile.NamedTemporaryFile(suffix='.xlsx', delete=False)
                xlsx_filename = language + "-words.xlsx"
                workbook = xlsxwriter.Workbook(temp_file.name)
                worksheet = workbook.add_worksheet()
                bold = workbook.add_format({'bold': 1, 'num_format': '@'})
                text = workbook.add_format({'num_format': '@'})
                text.set_align('top')
                wrapping = workbook.add_format({'num_format': '@'})
                wrapping.set_align('top')
                wrapping.set_text_wrap()
                wrapping.set_locked(False)
                numb = workbook.add_format()
                numb.set_align('top')
                worksheet.write('A1', 'orig_lang', bold)
                worksheet.write('B1', 'tr_lang', bold)
                worksheet.write('C1', 'orig_text', bold)
                worksheet.write('D1', 'tr_text', bold)
                worksheet.set_column(0, 0, 10)
                worksheet.set_column(1, 1, 10)
                worksheet.set_column(2, 2, 55)
                worksheet.set_column(3, 3, 55)
                row = 1
                for key, val in result[language].items():
                    worksheet.write_string(row, 0, 'en', text)
                    worksheet.write_string(row, 1, language, text)
                    worksheet.write_string(row, 2, key, wrapping)
                    worksheet.write_string(row, 3, val, wrapping)
                    row += 1
                workbook.close()
                response = send_file(temp_file.name, mimetype='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', as_attachment=True, attachment_filename=xlsx_filename)
                response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
                return response
            elif form.systemfiletype.data == 'XLIFF 1.2':
                temp_file = tempfile.NamedTemporaryFile(suffix='.xlf', delete=False)
                xliff_filename = language + "-words.xlf"
                xliff = ET.Element('xliff')
                xliff.set('xmlns', 'urn:oasis:names:tc:xliff:document:1.2')
                xliff.set('version', '1.2')
                the_file = ET.SubElement(xliff, 'file')
                the_file.set('source-language', 'en')
                the_file.set('target-language', language)
                the_file.set('datatype', 'plaintext')
                the_file.set('original', 'self')
                the_file.set('id', 'f1')
                the_file.set('xml:space', 'preserve')
                body = ET.SubElement(the_file, 'body')
                indexno = 1
                for key, val in result[language].items():
                    trans_unit = ET.SubElement(body, 'trans-unit')
                    trans_unit.set('id', str(indexno))
                    trans_unit.set('xml:space', 'preserve')
                    source = ET.SubElement(trans_unit, 'source')
                    source.set('xml:space', 'preserve')
                    target = ET.SubElement(trans_unit, 'target')
                    target.set('xml:space', 'preserve')
                    source.text = key
                    target.text = val
                    indexno += 1
                temp_file.write(ET.tostring(xliff))
                temp_file.close()
                response = send_file(temp_file.name, mimetype='application/xml', as_attachment=True, attachment_filename=xliff_filename)
                response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
                return response
            elif form.systemfiletype.data == 'XLIFF 2.0':
                temp_file = tempfile.NamedTemporaryFile(suffix='.xlf', delete=False)
                xliff_filename = language + "-words.xlf"
                xliff = ET.Element('xliff')
                xliff.set('xmlns', 'urn:oasis:names:tc:xliff:document:2.0')
                xliff.set('version', '2.0')
                xliff.set('srcLang', 'en')
                xliff.set('trgLang', language)
                file_index = 1
                the_file = ET.SubElement(xliff, 'file')
                the_file.set('id', 'f1')
                the_file.set('original', 'self')
                the_file.set('xml:space', 'preserve')
                unit = ET.SubElement(the_file, 'unit')
                unit.set('id', "docassemble_phrases")
                indexno = 1
                for key, val in result[language].items():
                    segment = ET.SubElement(unit, 'segment')
                    segment.set('id', str(indexno))
                    segment.set('xml:space', 'preserve')
                    source = ET.SubElement(segment, 'source')
                    source.set('xml:space', 'preserve')
                    target = ET.SubElement(segment, 'target')
                    target.set('xml:space', 'preserve')
                    source.text = key
                    target.text = val
                    indexno += 1
                temp_file.write(ET.tostring(xliff))
                temp_file.close()
                response = send_file(temp_file.name, mimetype='application/xml', as_attachment=True, attachment_filename=xliff_filename)
                response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
                return response
        if 'pdfdocxfile' in request.files and request.files['pdfdocxfile'].filename:
            filename = secure_filename(request.files['pdfdocxfile'].filename)
            extension, mimetype = get_ext_and_mimetype(filename)
            if mimetype == 'application/pdf':
                file_type = 'pdf'
                pdf_file = tempfile.NamedTemporaryFile(mode="wb", suffix=".pdf", delete=True)
                the_file = request.files['pdfdocxfile']
                the_file.save(pdf_file.name)
                try:
                    fields_output = read_fields(pdf_file.name, the_file.filename, 'pdf', 'yaml')
                except Exception as err:
                    fields_output = str(err)
                pdf_file.close()
            elif mimetype == 'application/vnd.openxmlformats-officedocument.wordprocessingml.document':
                file_type = 'docx'
                docx_file = tempfile.NamedTemporaryFile(mode="wb", suffix=".docx", delete=True)
                the_file = request.files['pdfdocxfile']
                the_file.save(docx_file.name)
                try:
                    fields_output = read_fields(docx_file.name, the_file.filename, 'docx', 'yaml')
                except Exception as err:
                    fields_output = str(err)
                docx_file.close()
        if form.officeaddin_submit.data:
            resp = make_response(render_template('pages/officemanifest.xml', office_app_version=form.officeaddin_version.data, guid=str(uuid.uuid4())))
            resp.headers['Content-type'] = 'text/xml; charset=utf-8'
            resp.headers['Content-Disposition'] = 'attachment; filename="manifest.xml"'
            resp.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
            return resp
    extra_js = """
    <script>
      $('#pdfdocxfile').on('change', function(){
        var fileName = $(this).val();
        fileName = fileName.replace(/.*\\\\/, '');
        fileName = fileName.replace(/.*\\//, '');
        $(this).next('.custom-file-label').html(fileName);
      });
    </script>"""
    form.systemfiletype.choices = [('YAML', 'YAML'), ('XLSX', 'XLSX'), ('XLIFF 1.2', 'XLIFF 1.2'), ('XLIFF 2.0', 'XLIFF 2.0')]
    form.systemfiletype.data = 'YAML'
    form.filetype.choices = [('XLSX', 'XLSX'), ('XLIFF 1.2', 'XLIFF 1.2'), ('XLIFF 2.0', 'XLIFF 2.0')]
    form.filetype.data = 'XLSX'
    response = make_response(render_template('pages/utilities.html', extra_js=Markup(extra_js), version_warning=version_warning, bodyclass='daadminbody', tab_title=word("Utilities"), page_title=word("Utilities"), form=form, fields=fields_output, word_box=word_box, uses_null=uses_null, file_type=file_type, interview_placeholder=word("E.g., docassemble.demo:data/questions/questions.yml"), language_placeholder=word("E.g., es, fr, it")), 200)
    response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
    return response

# @app.route('/save', methods=['GET', 'POST'])
# def save_for_later():
#     if current_user.is_authenticated and not current_user.is_anonymous:
#         return render_template('pages/save_for_later.html', interview=sdf)
#     secret = request.cookies.get('secret', None)

@app.route('/after_reset', methods=['GET', 'POST'])
def after_reset():
    #logmessage("after_reset")
    response = redirect(url_for('user.login'))
    if 'newsecret' in session:
        #logmessage("after_reset: fixing cookie")
        response.set_cookie('secret', session['newsecret'], httponly=True, secure=app.config['SESSION_COOKIE_SECURE'], samesite=app.config['SESSION_COOKIE_SAMESITE'])
        del session['newsecret']
    return response

# @app.before_request
# def reset_thread_local():
#     docassemble.base.functions.reset_thread_local()

# @app.after_request
# def remove_temporary_files(response):
#     docassemble.base.functions.close_files()
#     return response

def needs_to_change_password():
    if not current_user.has_role('admin'):
        return False
    if not (current_user.social_id and current_user.social_id.startswith('local')):
        return False
    #logmessage("needs_to_change_password: starting")
    if app.user_manager.verify_password('password', current_user):
        flash(word("Your password is insecure and needs to be changed"), "warning")
        return True
    #logmessage("needs_to_change_password: ending")
    return False

def fix_group_id(the_package, the_file, the_group_id):
    if the_package == '_global':
        group_id_to_use = the_group_id
    else:
        group_id_to_use = the_package
        if re.search(r'^data/', the_file):
            group_id_to_use += ':' + the_file
        else:
            group_id_to_use += ':data/sources/ml-' + the_file + '.json'
        group_id_to_use += ':' + the_group_id
    return group_id_to_use

def ensure_training_loaded(interview):
    # parts = yaml_filename.split(':')
    # if len(parts) != 2:
    #     logmessage("ensure_training_loaded: could not read yaml_filename " + str(yaml_filename))
    #     return
    # source_filename = parts[0] + ':data/sources/ml-' + re.sub(r'\.ya?ml$', '', re.sub(r'.*/', '', parts[1])) + '.json'
    #logmessage("Source filename is " + source_filename)
    source_filename = interview.get_ml_store()
    parts = source_filename.split(':')
    if len(parts) == 3 and parts[0].startswith('docassemble.') and re.match(r'data/sources/.*\.json', parts[1]):
        the_file = docassemble.base.functions.package_data_filename(source_filename)
        if the_file is not None:
            record = db.session.execute(select(MachineLearning.group_id).where(MachineLearning.group_id.like(source_filename + ':%'))).first()
            if record is None:
                if os.path.isfile(the_file):
                    with open(the_file, 'r', encoding='utf-8') as fp:
                        content = fp.read()
                    if len(content) > 0:
                        try:
                            href = json.loads(content)
                            if isinstance(href, dict):
                                nowtime = datetime.datetime.utcnow()
                                for group_id, train_list in href.items():
                                    if isinstance(train_list, list):
                                        for entry in train_list:
                                            if 'independent' in entry:
                                                new_entry = MachineLearning(group_id=source_filename + ':' + group_id, independent=codecs.encode(pickle.dumps(entry['independent']), 'base64').decode(), dependent=codecs.encode(pickle.dumps(entry.get('dependent', None)), 'base64').decode(), modtime=nowtime, create_time=nowtime, active=True, key=entry.get('key', None))
                                                db.session.add(new_entry)
                                db.session.commit()
                            else:
                                logmessage("ensure_training_loaded: source filename " + source_filename + " not used because it did not contain a dict")
                        except:
                            logmessage("ensure_training_loaded: source filename " + source_filename + " not used because it did not contain valid JSON")
                    else:
                        logmessage("ensure_training_loaded: source filename " + source_filename + " not used because its content was empty")
                else:
                    logmessage("ensure_training_loaded: source filename " + source_filename + " not used because it did not exist")
            else:
                logmessage("ensure_training_loaded: source filename " + source_filename + " not used because training data existed")
        else:
            logmessage("ensure_training_loaded: source filename " + source_filename + " did not exist")
    else:
        logmessage("ensure_training_loaded: source filename " + source_filename + " was not part of a package")

def get_corresponding_interview(the_package, the_file):
    #logmessage("get_corresponding_interview: " + the_package + " " + the_file)
    interview = None
    if re.match(r'docassemble.playground[0-9]+', the_package):
        separator = ':'
    else:
        separator = ':data/questions/'
    for interview_file in (the_package + separator + the_file + '.yml', the_package + separator + the_file + '.yaml', the_package + separator + 'examples/' + the_file + '.yml'):
        #logmessage("Looking for " + interview_file)
        try:
            interview = docassemble.base.interview_cache.get_interview(interview_file)
            break
        except Exception as the_err:
            #logmessage("There was an exception looking for " + interview_file + ": " + str(the_err))
            continue
    return interview

def ml_prefix(the_package, the_file):
    the_prefix = the_package
    if re.search(r'^data/', the_file):
        the_prefix += ':' + the_file
    else:
        the_prefix += ':data/sources/ml-' + the_file + '.json'
    return the_prefix

@app.route('/train', methods=['GET', 'POST'])
@login_required
@roles_required(['admin', 'developer', 'trainer'])
def train():
    setup_translation()
    the_package = request.args.get('package', None)
    if the_package is not None:
        the_package = werkzeug.utils.secure_filename(the_package)
    the_file = request.args.get('file', None)
    if the_file is not None:
        the_file = secure_filename_spaces_ok(the_file)
    the_group_id = request.args.get('group_id', None)
    show_all = int(request.args.get('show_all', 0))
    form = TrainingForm(request.form)
    uploadform = TrainingUploadForm(request.form)
    if request.method == 'POST' and the_package is not None and the_file is not None:
        if the_package == '_global':
            the_prefix = ''
        else:
            the_prefix = ml_prefix(the_package, the_file)
        json_file = None
        if the_package != '_global' and uploadform.usepackage.data == 'yes':
            the_file = docassemble.base.functions.package_data_filename(the_prefix)
            if the_file is None or not os.path.isfile(the_file):
                flash(word("Error reading JSON file from package.  File did not exist."), 'error')
                return redirect(url_for('train', package=the_package, file=the_file, group_id=the_group_id, show_all=show_all))
            json_file = open(the_file, 'r', encoding='utf-8')
        if uploadform.usepackage.data == 'no' and 'jsonfile' in request.files and request.files['jsonfile'].filename:
            json_file = tempfile.NamedTemporaryFile(prefix="datemp", suffix=".json")
            request.files['jsonfile'].save(json_file.name)
            json_file.seek(0)
        if json_file is not None:
            try:
                href = json.load(json_file)
            except:
                flash(word("Error reading JSON file.  Not a valid JSON file."), 'error')
                return redirect(url_for('train', package=the_package, file=the_file, group_id=the_group_id, show_all=show_all))
            json_file.close()
            if not isinstance(href, dict):
                flash(word("Error reading JSON file.  The JSON file needs to contain a dictionary at the root level."), 'error')
                return redirect(url_for('train', package=the_package, file=the_file, group_id=the_group_id, show_all=show_all))
            nowtime = datetime.datetime.utcnow()
            for group_id, train_list in href.items():
                if not isinstance(train_list, list):
                    logmessage("train: could not import part of JSON file.  Items in dictionary must be lists.")
                    continue
                if uploadform.importtype.data == 'replace':
                    db.session.execute(delete(MachineLearning).filter_by(group_id=the_prefix + ':' + group_id))
                    db.session.commit()
                    for entry in train_list:
                        if 'independent' in entry:
                            new_entry = MachineLearning(group_id=the_prefix + ':' + group_id, independent=codecs.encode(pickle.dumps(entry['independent']), 'base64').decode(), dependent=codecs.encode(pickle.dumps(entry.get('dependent', None)), 'base64').decode(), modtime=nowtime, create_time=nowtime, active=True, key=entry.get('key', None))
                            db.session.add(new_entry)
                elif uploadform.importtype.data == 'merge':
                    indep_in_use = set()
                    for record in db.session.execute(select(MachineLearning).filter_by(group_id=the_prefix + ':' + group_id)).scalars():
                        indep_in_use.add(fix_pickle_obj(codecs.decode(bytearray(record.independent, encoding='utf-8'), 'base64')))
                    for entry in train_list:
                        if 'independent' in entry and entry['independent'] not in indep_in_use:
                            new_entry = MachineLearning(group_id=the_prefix + ':' + group_id, independent=codecs.encode(pickle.dumps(entry['independent']), 'base64').decode(), dependent=codecs.encode(pickle.dumps(entry.get('dependent', None)), 'base64').decode(), modtime=nowtime, create_time=nowtime, active=True, key=entry.get('key', None))
                            db.session.add(new_entry)
            db.session.commit()
            flash(word("Training data were successfully imported."), 'success')
            return redirect(url_for('train', package=the_package, file=the_file, group_id=the_group_id, show_all=show_all))
        if form.cancel.data:
            return redirect(url_for('train', package=the_package, file=the_file, show_all=show_all))
        if form.submit.data:
            group_id_to_use = fix_group_id(the_package, the_file, the_group_id)
            post_data = request.form.copy()
            deleted = set()
            for key, val in post_data.items():
                m = re.match(r'delete([0-9]+)', key)
                if not m:
                    continue
                entry_id = int(m.group(1))
                model = docassemble.base.util.SimpleTextMachineLearner(group_id=group_id_to_use)
                model.delete_by_id(entry_id)
                deleted.add('dependent' + m.group(1))
            for key in deleted:
                if key in post_data:
                    del post_data[key]
            for key, val in post_data.items():
                m = re.match(r'dependent([0-9]+)', key)
                if not m:
                    continue
                orig_key = 'original' + m.group(1)
                delete_key = 'delete' + m.group(1)
                if orig_key in post_data and post_data[orig_key] != val and val != '':
                    entry_id = int(m.group(1))
                    model = docassemble.base.util.SimpleTextMachineLearner(group_id=group_id_to_use)
                    model.set_dependent_by_id(entry_id, val)
            if post_data.get('newindependent', '') != '':
                model = docassemble.base.util.SimpleTextMachineLearner(group_id=group_id_to_use)
                if post_data.get('newdependent', '') != '':
                    model.add_to_training_set(post_data['newindependent'], post_data['newdependent'])
                else:
                    model.save_for_classification(post_data['newindependent'])
            return redirect(url_for('train', package=the_package, file=the_file, group_id=the_group_id, show_all=show_all))
    if show_all:
        show_all = 1
        show_cond = MachineLearning.id != None
    else:
        show_all = 0
        show_cond = MachineLearning.dependent is None
    package_list = {}
    file_list = {}
    group_id_list = {}
    entry_list = []
    if current_user.has_role('admin', 'developer'):
        playground_package = 'docassemble.playground' + str(current_user.id)
    else:
        playground_package = None
    if the_package is None:
        for record in db.session.execute(select(MachineLearning.group_id, db.func.count(MachineLearning.id).label('cnt')).where(show_cond).group_by(MachineLearning.group_id)):
            group_id = record.group_id
            parts = group_id.split(':')
            if is_package_ml(parts):
                if parts[0] not in package_list:
                    package_list[parts[0]] = 0
                package_list[parts[0]] += record.cnt
            else:
                if '_global' not in package_list:
                    package_list['_global'] = 0
                package_list['_global'] += record.cnt
        if not show_all:
            for record in db.session.execute(select(MachineLearning.group_id).group_by(MachineLearning.group_id)):
                parts = record.group_id.split(':')
                if is_package_ml(parts):
                    if parts[0] not in package_list:
                        package_list[parts[0]] = 0
            if '_global' not in package_list:
                package_list['_global'] = 0
        if playground_package and playground_package not in package_list:
            package_list[playground_package] = 0
        package_list = [(x, package_list[x]) for x in sorted(package_list)]
        response = make_response(render_template('pages/train.html', version_warning=version_warning, bodyclass='daadminbody', tab_title=word("Train"), page_title=word("Train machine learning models"), the_package=the_package, the_file=the_file, the_group_id=the_group_id, package_list=package_list, file_list=file_list, group_id_list=group_id_list, entry_list=entry_list, show_all=show_all, show_package_list=True, playground_package=playground_package), 200)
        response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
        return response
    if playground_package and the_package == playground_package:
        the_package_display = word("My Playground")
    else:
        the_package_display = the_package
    if the_file is None:
        file_list = {}
        for record in db.session.execute(select(MachineLearning.group_id, db.func.count(MachineLearning.id).label('cnt')).where(and_(MachineLearning.group_id.like(the_package + ':%'), show_cond)).group_by(MachineLearning.group_id)):
            parts = record.group_id.split(':')
            #logmessage("Group id is " + str(parts))
            if not is_package_ml(parts):
                continue
            if re.match(r'data/sources/ml-.*\.json', parts[1]):
                parts[1] = re.sub(r'^data/sources/ml-|\.json$', '', parts[1])
            if parts[1] not in file_list:
                file_list[parts[1]] = 0
            file_list[parts[1]] += record.cnt
        if not show_all:
            for record in db.session.execute(select(MachineLearning.group_id).where(MachineLearning.group_id.like(the_package + ':%')).group_by(MachineLearning.group_id)):
                parts = record.group_id.split(':')
                #logmessage("Other group id is " + str(parts))
                if not is_package_ml(parts):
                    continue
                if re.match(r'data/sources/ml-.*\.json', parts[1]):
                    parts[1] = re.sub(r'^data/sources/ml-|\.json$', '', parts[1])
                if parts[1] not in file_list:
                    file_list[parts[1]] = 0
        if playground_package and the_package == playground_package:
            area = SavedFile(current_user.id, fix=False, section='playgroundsources')
            for filename in area.list_of_files():
                #logmessage("hey file is " + str(filename))
                if re.match(r'ml-.*\.json', filename):
                    short_file_name = re.sub(r'^ml-|\.json$', '', filename)
                    if short_file_name not in file_list:
                        file_list[short_file_name] = 0
        file_list = [(x, file_list[x]) for x in sorted(file_list)]
        response = make_response(render_template('pages/train.html', version_warning=version_warning, bodyclass='daadminbody', tab_title=word("Train"), page_title=word("Train machine learning models"), the_package=the_package, the_package_display=the_package_display, the_file=the_file, the_group_id=the_group_id, package_list=package_list, file_list=file_list, group_id_list=group_id_list, entry_list=entry_list, show_all=show_all, show_file_list=True), 200)
        response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
        return response
    if the_group_id is None:
        the_prefix = ml_prefix(the_package, the_file)
        the_package_file = docassemble.base.functions.package_data_filename(the_prefix)
        package_file_available = bool(the_package_file is not None and os.path.isfile(the_package_file))
        if 'download' in request.args and request.args['download']:
            output = {}
            if the_package == '_global':
                json_filename = 'ml-global.json'
                for record in db.session.execute(select(MachineLearning.id, MachineLearning.group_id, MachineLearning.independent, MachineLearning.dependent, MachineLearning.key)):
                    if is_package_ml(record.group_id.split(':')):
                        continue
                    if record.group_id not in output:
                        output[record.group_id] = []
                    if record.dependent is None:
                        the_dependent = None
                    else:
                        the_dependent = fix_pickle_obj(codecs.decode(bytearray(record.dependent, encoding='utf-8'), 'base64'))
                    the_independent = fix_pickle_obj(codecs.decode(bytearray(record.independent, encoding='utf-8'), 'base64'))
                    try:
                        str(the_independent) + ""
                        str(the_dependent) + ""
                    except Exception as e:
                        logmessage("Bad record: id " + str(record.id) + " where error was " + str(e))
                        continue
                    the_entry = dict(independent=fix_pickle_obj(codecs.decode(bytearray(record.independent, encoding='utf-8'), 'base64')), dependent=the_dependent)
                    if record.key is not None:
                        the_entry['key'] = record.key
                    output[record.group_id].append(the_entry)
            else:
                json_filename = 'ml-' + the_file + '.json'
                prefix = ml_prefix(the_package, the_file)
                for record in db.session.execute(select(MachineLearning.group_id, MachineLearning.independent, MachineLearning.dependent, MachineLearning.key).where(MachineLearning.group_id.like(prefix + ':%'))):
                    parts = record.group_id.split(':')
                    if not is_package_ml(parts):
                        continue
                    if parts[2] not in output:
                        output[parts[2]] = []
                    if record.dependent is None:
                        the_dependent = None
                    else:
                        the_dependent = fix_pickle_obj(codecs.decode(bytearray(record.dependent, encoding='utf-8'), 'base64'))
                    the_entry = dict(independent=fix_pickle_obj(codecs.decode(bytearray(record.independent, encoding='utf-8'), 'base64')), dependent=the_dependent)
                    if record.key is not None:
                        the_entry['key'] = record.key
                    output[parts[2]].append(the_entry)
            if len(output) > 0:
                the_json_file = tempfile.NamedTemporaryFile(mode='w', prefix="datemp", suffix=".json", delete=False, encoding='utf-8')
                json.dump(output, the_json_file, sort_keys=True, indent=2)
                response = send_file(the_json_file, mimetype='application/json', as_attachment=True, attachment_filename=json_filename)
                response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
                return response
            else:
                flash(word("No data existed in training set.  JSON file not created."), "error")
                return redirect(url_for('train', package=the_package, file=the_file, show_all=show_all))
        if the_package == '_global':
            for record in db.session.execute(select(MachineLearning.group_id, db.func.count(MachineLearning.id).label('cnt')).where(show_cond).group_by(MachineLearning.group_id)):
                if is_package_ml(record.group_id.split(':')):
                    continue
                if record.group_id not in group_id_list:
                    group_id_list[record.group_id] = 0
                group_id_list[record.group_id] += record.cnt
            if not show_all:
                for record in db.session.execute(select(MachineLearning.group_id).group_by(MachineLearning.group_id)):
                    if is_package_ml(record.group_id.split(':')):
                        continue
                    if record.group_id not in group_id_list:
                        group_id_list[record.group_id] = 0
        else:
            the_prefix = ml_prefix(the_package, the_file)
            #logmessage("My prefix is " + the_prefix)
            for record in db.session.execute(select(MachineLearning.group_id, db.func.count(MachineLearning.id).label('cnt')).where(and_(MachineLearning.group_id.like(the_prefix + ':%'), show_cond)).group_by(MachineLearning.group_id)):
                parts = record.group_id.split(':')
                if not is_package_ml(parts):
                    continue
                if parts[2] not in group_id_list:
                    group_id_list[parts[2]] = 0
                group_id_list[parts[2]] += record.cnt
            if not show_all:
                for record in db.session.execute(select(MachineLearning.group_id).where(MachineLearning.group_id.like(the_prefix + ':%')).group_by(MachineLearning.group_id)):
                    parts = record.group_id.split(':')
                    if not is_package_ml(parts):
                        continue
                    if parts[2] not in group_id_list:
                        group_id_list[parts[2]] = 0
        if the_package != '_global' and not re.search(r'^data/', the_file):
            interview = get_corresponding_interview(the_package, the_file)
            if interview is not None and len(interview.mlfields):
                for saveas in interview.mlfields:
                    if 'ml_group' in interview.mlfields[saveas] and not interview.mlfields[saveas]['ml_group'].uses_mako:
                        the_saveas = interview.mlfields[saveas]['ml_group'].original_text
                    else:
                        the_saveas = saveas
                    if not re.search(r':', the_saveas):
                        if the_saveas not in group_id_list:
                            group_id_list[the_saveas] = 0
        group_id_list = [(x, group_id_list[x]) for x in sorted(group_id_list)]
        extra_js = """
    <script>
      $( document ).ready(function() {
        $("#showimport").click(function(e){
          $("#showimport").hide();
          $("#hideimport").show();
          $("#importcontrols").show('fast');
          e.preventDefault();
          return false;
        });
        $("#hideimport").click(function(e){
          $("#showimport").show();
          $("#hideimport").hide();
          $("#importcontrols").hide('fast');
          e.preventDefault();
          return false;
        });
        $("input[type=radio][name=usepackage]").on('change', function(e) {
          if ($(this).val() == 'no'){
            $("#uploadinput").show();
          }
          else{
            $("#uploadinput").hide();
          }
          e.preventDefault();
          return false;
        });
      });
    </script>"""
        response = make_response(render_template('pages/train.html', extra_js=Markup(extra_js), version_warning=version_warning, bodyclass='daadminbody', tab_title=word("Train"), page_title=word("Train machine learning models"), the_package=the_package, the_package_display=the_package_display, the_file=the_file, the_group_id=the_group_id, package_list=package_list, file_list=file_list, group_id_list=group_id_list, entry_list=entry_list, show_all=show_all, show_group_id_list=True, package_file_available=package_file_available, the_package_location=the_prefix, uploadform=uploadform), 200)
        response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
        return response
    else:
        group_id_to_use = fix_group_id(the_package, the_file, the_group_id)
        model = docassemble.base.util.SimpleTextMachineLearner(group_id=group_id_to_use)
        for record in db.session.execute(select(MachineLearning.id, MachineLearning.group_id, MachineLearning.key, MachineLearning.info, MachineLearning.independent, MachineLearning.dependent, MachineLearning.create_time, MachineLearning.modtime, MachineLearning.active).where(and_(MachineLearning.group_id == group_id_to_use, show_cond))):
            new_entry = dict(id=record.id, group_id=record.group_id, key=record.key, independent=fix_pickle_obj(codecs.decode(bytearray(record.independent, encoding='utf-8'), 'base64')) if record.independent is not None else None, dependent=fix_pickle_obj(codecs.decode(bytearray(record.dependent, encoding='utf-8'), 'base64')) if record.dependent is not None else None, info=fix_pickle_obj(codecs.decode(bytearray(record.info, encoding='utf-8'), 'base64')) if record.info is not None else None, create_type=record.create_time, modtime=record.modtime, active=MachineLearning.active)
            if isinstance(new_entry['independent'], DADict) or isinstance(new_entry['independent'], dict):
                new_entry['independent_display'] = '<div class="damldatacontainer">' + '<br>'.join(['<span class="damldatakey">' + str(key) + '</span>: <span class="damldatavalue">' + str(val) + ' (' + str(val.__class__.__name__) + ')</span>' for key, val in new_entry['independent'].items()]) + '</div>'
                new_entry['type'] = 'data'
            else:
                new_entry['type'] = 'text'
            if new_entry['dependent'] is None:
                new_entry['predictions'] = model.predict(new_entry['independent'], probabilities=True)
                if len(new_entry['predictions']) == 0:
                    new_entry['predictions'] = None
                elif len(new_entry['predictions']) > 10:
                    new_entry['predictions'] = new_entry['predictions'][0:10]
                if new_entry['predictions'] is not None:
                    new_entry['predictions'] = [(prediction, '%d%%' % (100.0*probability)) for prediction, probability in new_entry['predictions']]
            else:
                new_entry['predictions'] = None
            if new_entry['info'] is not None:
                if isinstance(new_entry['info'], DAFile):
                    image_file_list = [new_entry['info']]
                elif isinstance(new_entry['info'], DAFileList):
                    image_file_list = new_entry['info']
                else:
                    logmessage("train: info is not a DAFile or DAFileList")
                    continue
                new_entry['image_files'] = []
                for image_file in image_file_list:
                    if not isinstance(image_file, DAFile):
                        logmessage("train: file is not a DAFile")
                        continue
                    if not image_file.ok:
                        logmessage("train: file does not have a number")
                        continue
                    if image_file.extension not in ('pdf', 'png', 'jpg', 'jpeg', 'gif'):
                        logmessage("train: file did not have a recognizable image type")
                        continue
                    doc_url = get_url_from_file_reference(image_file)
                    if image_file.extension == 'pdf':
                        image_url = get_url_from_file_reference(image_file, size="screen", page=1, ext='pdf')
                    else:
                        image_url = doc_url
                    new_entry['image_files'].append(dict(doc_url=doc_url, image_url=image_url))
            entry_list.append(new_entry)
        if len(entry_list) == 0:
            record = db.session.execute(select(MachineLearning.independent).where(and_(MachineLearning.group_id == group_id_to_use, MachineLearning.independent != None))).first()
            if record is not None:
                sample_indep = fix_pickle_obj(codecs.decode(bytearray(record.independent, encoding='utf-8'), 'base64'))
            else:
                sample_indep = None
        else:
            sample_indep = entry_list[0]['independent']
        is_data = bool(isinstance(sample_indep, DADict) or isinstance(sample_indep, dict))
        choices = {}
        for record in db.session.execute(select(MachineLearning.dependent, db.func.count(MachineLearning.id).label('cnt')).where(and_(MachineLearning.group_id == group_id_to_use)).group_by(MachineLearning.dependent)):
            #logmessage("There is a choice")
            if record.dependent is None:
                continue
            key = fix_pickle_obj(codecs.decode(bytearray(record.dependent, encoding='utf-8'), 'base64'))
            choices[key] = record.cnt
        if len(choices) > 0:
            #logmessage("There are choices")
            choices = [(x, choices[x]) for x in sorted(choices, key=operator.itemgetter(0), reverse=False)]
        else:
            #logmessage("There are no choices")
            choices = None
        extra_js = """
    <script>
      $( document ).ready(function(){
        $("button.prediction").click(function(){
          if (!($("#dependent" + $(this).data("id-number")).prop('disabled'))){
            $("#dependent" + $(this).data("id-number")).val($(this).data("prediction"));
          }
        });
        $("select.trainer").change(function(){
          var the_number = $(this).data("id-number");
          if (the_number == "newdependent"){
            $("#newdependent").val($(this).val());
          }
          else{
            $("#dependent" + the_number).val($(this).val());
          }
        });
        $("div.dadelete-observation input").change(function(){
          var the_number = $(this).data("id-number");
          if ($(this).is(':checked')){
            $("#dependent" + the_number).prop('disabled', true);
            $("#selector" + the_number).prop('disabled', true);
          }
          else{
            $("#dependent" + the_number).prop('disabled', false);
            $("#selector" + the_number).prop('disabled', false);
          }
        });
      });
    </script>"""
        response = make_response(render_template('pages/train.html', extra_js=Markup(extra_js), form=form, version_warning=version_warning, bodyclass='daadminbody', tab_title=word("Train"), page_title=word("Train machine learning models"), the_package=the_package, the_package_display=the_package_display, the_file=the_file, the_group_id=the_group_id, entry_list=entry_list, choices=choices, show_all=show_all, show_entry_list=True, is_data=is_data), 200)
        response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
        return response

def user_interviews(user_id=None, secret=None, exclude_invalid=True, action=None, filename=None, session=None, tag=None, include_dict=True, delete_shared=False, admin=False, start_id=None, temp_user_id=None):
    # logmessage("user_interviews: user_id is " + str(user_id) + " and secret is " + str(secret))
    if user_id is None and (current_user.is_anonymous or not current_user.has_role('admin', 'advocate')):
        raise Exception('user_interviews: only administrators and advocates can access information about other users')
    if user_id is not None and admin is False and not current_user.is_anonymous and current_user.id != user_id and not current_user.has_role('admin', 'advocate'):
        raise Exception('user_interviews: only administrators and advocates can access information about other users')
    if action is not None and admin is False and not current_user.has_role('admin', 'advocate'):
        if user_id is None:
            raise Exception("user_interviews: no user_id provided")
        the_user = get_person(int(user_id), {})
        if the_user is None:
            raise Exception("user_interviews: user_id " + str(user_id) + " not valid")
    if action == 'delete_all':
        sessions_to_delete = set()
        if tag:
            start_id = None
            while True:
                (the_list, start_id) = user_interviews(user_id=user_id, secret=secret, tag=tag, include_dict=False, start_id=start_id, temp_user_id=temp_user_id)
                for interview_info in the_list:
                    sessions_to_delete.add((interview_info['session'], interview_info['filename'], interview_info['user_id'], interview_info['temp_user_id']))
                if start_id is None:
                    break
        else:
            if temp_user_id is not None:
                if filename is None:
                    interview_query = db.session.execute(select(UserDictKeys.filename, UserDictKeys.key).where(UserDictKeys.temp_user_id == temp_user_id).group_by(UserDictKeys.filename, UserDictKeys.key))
                else:
                    interview_query = db.session.execute(select(UserDictKeys.filename, UserDictKeys.key).where(UserDictKeys.temp_user_id == temp_user_id, UserDictKeys.filename == filename).group_by(UserDictKeys.filename, UserDictKeys.key))
                for interview_info in interview_query:
                    sessions_to_delete.add((interview_info.key, interview_info.filename, None, temp_user_id))
            elif user_id is not None:
                if filename is None:
                    interview_query = db.session.execute(select(UserDictKeys.filename, UserDictKeys.key).where(UserDictKeys.user_id == user_id).group_by(UserDictKeys.filename, UserDictKeys.key))
                else:
                    interview_query = db.session.execute(select(UserDictKeys.filename, UserDictKeys.key).where(UserDictKeys.user_id == user_id, UserDictKeys.filename == filename).group_by(UserDictKeys.filename, UserDictKeys.key))
                for interview_info in interview_query:
                    sessions_to_delete.add((interview_info.key, interview_info.filename, user_id, None))
            else:
                if filename is None:
                    interview_query = db.session.execute(select(UserDict.filename, UserDict.key).group_by(UserDict.filename, UserDict.key))
                else:
                    interview_query = db.session.execute(select(UserDict.filename, UserDict.key).where(UserDict.filename == filename).group_by(UserDict.filename, UserDict.key))
                for interview_info in interview_query:
                    sessions_to_delete.add((interview_info.key, interview_info.filename, None, None))
            if user_id is not None:
                if filename is None:
                    interview_query = db.session.execute(select(UserDict.filename, UserDict.key).where(UserDict.user_id == user_id).group_by(UserDict.filename, UserDict.key))
                else:
                    interview_query = db.session.execute(select(UserDict.filename, UserDict.key).where(UserDict.user_id == user_id, UserDict.filename == filename).group_by(UserDict.filename, UserDict.key))
                for interview_info in interview_query:
                    sessions_to_delete.add((interview_info.key, interview_info.filename, user_id, None))
        logmessage("Deleting " + str(len(sessions_to_delete)) + " interviews")
        if len(sessions_to_delete) > 0:
            for session_id, yaml_filename, the_user_id, the_temp_user_id in sessions_to_delete:
                manual_checkout(manual_session_id=session_id, manual_filename=yaml_filename, user_id=the_user_id, delete_session=True, temp_user_id=the_temp_user_id)
                #obtain_lock(session_id, yaml_filename)
                if the_user_id is None or delete_shared:
                    reset_user_dict(session_id, yaml_filename, user_id=the_user_id, temp_user_id=the_temp_user_id, force=True)
                else:
                    reset_user_dict(session_id, yaml_filename, user_id=the_user_id, temp_user_id=the_temp_user_id)
                #release_lock(session_id, yaml_filename)
        return len(sessions_to_delete)
    if action == 'delete':
        if filename is None or session is None:
            raise Exception("user_interviews: filename and session must be provided in order to delete interview")
        manual_checkout(manual_session_id=session, manual_filename=filename, user_id=user_id, temp_user_id=temp_user_id, delete_session=True)
        #obtain_lock(session, filename)
        reset_user_dict(session, filename, user_id=user_id, temp_user_id=temp_user_id, force=delete_shared)
        #release_lock(session, filename)
        return True
    if admin is False and current_user and current_user.is_authenticated and current_user.timezone:
        the_timezone = pytz.timezone(current_user.timezone)
    else:
        the_timezone = pytz.timezone(get_default_timezone())

    interviews_length = 0
    interviews = []

    while True:
        there_are_more = False
        if temp_user_id is not None:
            query_elements = [UserDict.indexno, UserDictKeys.user_id, UserDictKeys.temp_user_id, UserDictKeys.filename, UserDictKeys.key, UserModel.email]
            subq_filter_elements = [UserDictKeys.temp_user_id == temp_user_id]
            if include_dict:
                query_elements.extend([UserDict.dictionary, UserDict.encrypted, UserModel.email])
            else:
                query_elements.append(UserDict.modtime)
            if filename is not None:
                subq_filter_elements.append(UserDictKeys.filename == filename)
            if session is not None:
                subq_filter_elements.append(UserDictKeys.key == session)
            if start_id is not None:
                subq_filter_elements.append(UserDict.indexno > start_id)
            subq = select(UserDictKeys.filename, UserDictKeys.key, db.func.max(UserDict.indexno).label('indexno')).join(UserDict, and_(UserDictKeys.filename == UserDict.filename, UserDictKeys.key == UserDict.key))
            if len(subq_filter_elements) > 0:
                subq = subq.where(and_(*subq_filter_elements))
            subq = subq.group_by(UserDictKeys.filename, UserDictKeys.key).subquery()
            interview_query = select(*query_elements).select_from(subq.join(UserDict, subq.c.indexno == UserDict.indexno).join(UserDictKeys, and_(UserDict.filename == UserDictKeys.filename, UserDict.key == UserDictKeys.key)).outerjoin(UserModel, 0 == 1)).order_by(UserDict.indexno)
        elif user_id is not None:
            query_elements = [UserDict.indexno, UserDictKeys.user_id, UserDictKeys.temp_user_id, UserDictKeys.filename, UserDictKeys.key, UserModel.email]
            subq_filter_elements = [UserDictKeys.user_id == user_id]
            if include_dict:
                query_elements.extend([UserDict.dictionary, UserDict.encrypted, UserModel.email])
            else:
                query_elements.append(UserDict.modtime)
            if filename is not None:
                subq_filter_elements.append(UserDictKeys.filename == filename)
            if session is not None:
                subq_filter_elements.append(UserDictKeys.key == session)
            if start_id is not None:
                subq_filter_elements.append(UserDict.indexno > start_id)
            subq = select(UserDictKeys.filename, UserDictKeys.key, db.func.max(UserDict.indexno).label('indexno')).join(UserDict, and_(UserDictKeys.filename == UserDict.filename, UserDictKeys.key == UserDict.key))
            if len(subq_filter_elements) > 0:
                subq = subq.where(and_(*subq_filter_elements))
            subq = subq.group_by(UserDictKeys.filename, UserDictKeys.key).subquery()
            interview_query = select(*query_elements).select_from(subq.join(UserDict, subq.c.indexno == UserDict.indexno).join(UserDictKeys, and_(UserDict.filename == UserDictKeys.filename, UserDict.key == UserDictKeys.key)).join(UserModel, UserDictKeys.user_id == UserModel.id)).order_by(UserDict.indexno)
        else:
            query_elements = [UserDict.indexno, UserDictKeys.user_id, UserDictKeys.temp_user_id, UserDict.filename, UserDict.key, UserModel.email]
            subq_filter_elements = []
            if include_dict:
                query_elements.extend([UserDict.dictionary, UserDict.encrypted, UserModel.email])
            else:
                query_elements.append(UserDict.modtime)
            if filename is not None:
                subq_filter_elements.append(UserDict.filename == filename)
            if session is not None:
                subq_filter_elements.append(UserDict.key == session)
            if start_id is not None:
                subq_filter_elements.append(UserDict.indexno > start_id)
            subq = select(UserDict.filename, UserDict.key, db.func.max(UserDict.indexno).label('indexno'))
            if len(subq_filter_elements) > 0:
                subq = subq.where(and_(*subq_filter_elements))
            subq = subq.group_by(UserDict.filename, UserDict.key).subquery()
            interview_query = select(*query_elements).select_from(subq.join(UserDict, subq.c.indexno == UserDict.indexno).join(UserDictKeys, and_(UserDict.filename == UserDictKeys.filename, UserDict.key == UserDictKeys.key)).outerjoin(UserModel, and_(UserDictKeys.user_id == UserModel.id, UserModel.active == True))).order_by(UserDict.indexno)
        interview_query = interview_query.limit(PAGINATION_LIMIT_PLUS_ONE)
        stored_info = []
        results_in_query = 0
        for interview_info in db.session.execute(interview_query):
            results_in_query += 1
            if results_in_query == PAGINATION_LIMIT_PLUS_ONE:
                there_are_more = True
                break
            #logmessage("filename is " + str(interview_info.filename) + " " + str(interview_info.key))
            if session is not None and interview_info.key != session:
                continue
            if include_dict and interview_info.dictionary is None:
                continue
            if include_dict:
                stored_info.append(dict(filename=interview_info.filename,
                                        encrypted=interview_info.encrypted,
                                        dictionary=interview_info.dictionary,
                                        key=interview_info.key,
                                        email=interview_info.email,
                                        user_id=interview_info.user_id,
                                        temp_user_id=interview_info.temp_user_id,
                                        indexno=interview_info.indexno))
            else:
                stored_info.append(dict(filename=interview_info.filename,
                                        modtime=interview_info.modtime,
                                        key=interview_info.key,
                                        email=interview_info.email,
                                        user_id=interview_info.user_id,
                                        temp_user_id=interview_info.temp_user_id,
                                        indexno=interview_info.indexno))
        for interview_info in stored_info:
            if interviews_length == PAGINATION_LIMIT:
                there_are_more = True
                break
            start_id = interview_info['indexno']
            interview_title = {}
            is_valid = True
            interview_valid = True
            try:
                interview = docassemble.base.interview_cache.get_interview(interview_info['filename'])
            except Exception as the_err:
                if exclude_invalid:
                    continue
                logmessage("user_interviews: unable to load interview file " + interview_info['filename'])
                interview_title['full'] = word('Error: interview not found')
                interview_valid = False
                is_valid = False
            #logmessage("Found old interview with title " + interview_title)
            if include_dict:
                if interview_info['encrypted']:
                    try:
                        dictionary = decrypt_dictionary(interview_info['dictionary'], secret)
                    except Exception as the_err:
                        if exclude_invalid:
                            continue
                        try:
                            logmessage("user_interviews: unable to decrypt dictionary.  " + str(the_err.__class__.__name__) + ": " + str(the_err))
                        except:
                            logmessage("user_interviews: unable to decrypt dictionary.  " + str(the_err.__class__.__name__))
                        dictionary = fresh_dictionary()
                        dictionary['_internal']['starttime'] = None
                        dictionary['_internal']['modtime'] = None
                        is_valid = False
                else:
                    try:
                        dictionary = unpack_dictionary(interview_info['dictionary'])
                    except Exception as the_err:
                        if exclude_invalid:
                            continue
                        try:
                            logmessage("user_interviews: unable to unpack dictionary.  " + str(the_err.__class__.__name__) + ": " + str(the_err))
                        except:
                            logmessage("user_interviews: unable to unpack dictionary.  " + str(the_err.__class__.__name__))
                        dictionary = fresh_dictionary()
                        dictionary['_internal']['starttime'] = None
                        dictionary['_internal']['modtime'] = None
                        is_valid = False
                if not isinstance(dictionary, dict):
                    logmessage("user_interviews: found a dictionary that was not a dictionary")
                    continue
            if is_valid:
                if include_dict:
                    interview_title = interview.get_title(dictionary)
                    tags = interview.get_tags(dictionary)
                else:
                    interview_title = interview.get_title(dict(_internal={}))
                    tags = interview.get_tags(dict(_internal={}))
                metadata = copy.deepcopy(interview.consolidated_metadata)
            elif interview_valid:
                interview_title = interview.get_title(dict(_internal={}))
                metadata = copy.deepcopy(interview.consolidated_metadata)
                if include_dict:
                    tags = interview.get_tags(dictionary)
                    if 'full' not in interview_title:
                        interview_title['full'] = word("Interview answers cannot be decrypted")
                    else:
                        interview_title['full'] += ' - ' + word('interview answers cannot be decrypted')
                else:
                    tags = interview.get_tags(dict(_internal={}))
                    if 'full' not in interview_title:
                        interview_title['full'] = word('Unknown')
            else:
                interview_title['full'] = word('Error: interview not found and answers could not be decrypted')
                metadata = {}
                tags = set()
            if include_dict:
                if dictionary['_internal']['starttime']:
                    utc_starttime = dictionary['_internal']['starttime']
                    starttime = nice_date_from_utc(dictionary['_internal']['starttime'], timezone=the_timezone)
                else:
                    utc_starttime = None
                    starttime = ''
                if dictionary['_internal']['modtime']:
                    utc_modtime = dictionary['_internal']['modtime']
                    modtime = nice_date_from_utc(dictionary['_internal']['modtime'], timezone=the_timezone)
                else:
                    utc_modtime = None
                    modtime = ''
            else:
                utc_starttime = None
                starttime = ''
                utc_modtime = interview_info['modtime']
                modtime = nice_date_from_utc(interview_info['modtime'], timezone=the_timezone)
            if tag is not None and tag not in tags:
                continue
            out = {'filename': interview_info['filename'], 'session': interview_info['key'], 'modtime': modtime, 'starttime': starttime, 'utc_modtime': utc_modtime, 'utc_starttime': utc_starttime, 'title': interview_title.get('full', word('Untitled')), 'subtitle': interview_title.get('sub', None), 'valid': is_valid, 'metadata': metadata, 'tags': tags, 'email': interview_info['email'], 'user_id': interview_info['user_id'], 'temp_user_id': interview_info['temp_user_id']}
            if include_dict:
                out['dict'] = dictionary
                out['encrypted'] = interview_info['encrypted']
            interviews.append(out)
            interviews_length += 1
        if interviews_length == PAGINATION_LIMIT or results_in_query < PAGINATION_LIMIT_PLUS_ONE:
            break
    if there_are_more:
        return (interviews, start_id)
    else:
        return (interviews, None)

@app.route('/interviews', methods=['GET', 'POST'])
@login_required
def interview_list():
    setup_translation()
    is_json = bool(('json' in request.form and as_int(request.form['json'])) or ('json' in request.args and as_int(request.args['json'])))
    if 'lang' in request.form:
        session['language'] = request.form['lang']
        docassemble.base.functions.set_language(session['language'])
    tag = request.args.get('tag', None)
    if tag is not None:
        tag = werkzeug.utils.secure_filename(tag)
    if 'newsecret' in session:
        #logmessage("interview_list: fixing cookie")
        the_args = {}
        if is_json:
            the_args['json'] = '1'
        if tag:
            the_args['tag'] = tag
        if 'from_login' in request.args:
            the_args['from_login'] = request.args['from_login']
        if 'post_restart' in request.args:
            the_args['post_restart'] = request.args['post_restart']
        if 'resume' in request.args:
            the_args['resume'] = request.args['resume']
        response = redirect(url_for('interview_list', **the_args))
        response.set_cookie('secret', session['newsecret'], httponly=True, secure=app.config['SESSION_COOKIE_SECURE'], samesite=app.config['SESSION_COOKIE_SAMESITE'])
        del session['newsecret']
        return response
    if request.method == 'GET' and needs_to_change_password():
        return redirect(url_for('user.change_password', next=url_for('interview_list')))
    secret = request.cookies.get('secret', None)
    if secret is not None:
        secret = str(secret)
    #logmessage("interview_list: secret is " + repr(secret))
    if 'action' in request.args and request.args.get('action') == 'delete_all':
        num_deleted = user_interviews(user_id=current_user.id, secret=secret, action='delete_all', tag=tag)
        if num_deleted > 0:
            flash(word("Deleted interviews"), 'success')
        if is_json:
            return redirect(url_for('interview_list', json='1'))
        return redirect(url_for('interview_list'))
    elif 'action' in request.args and request.args.get('action') == 'delete':
        yaml_file = request.args.get('filename', None)
        session_id = request.args.get('session', None)
        if yaml_file is not None and session_id is not None:
            user_interviews(user_id=current_user.id, secret=secret, action='delete', session=session_id, filename=yaml_file)
            flash(word("Deleted interview"), 'success')
        if is_json:
            return redirect(url_for('interview_list', json='1'))
        return redirect(url_for('interview_list'))
    #if daconfig.get('resume interview after login', False) and 'i' in session and 'uid' in session and (request.args.get('from_login', False) or (re.search(r'user/(register|sign-in)', str(request.referrer)) and 'next=' not in str(request.referrer))):
    #    if is_json:
    #        return redirect(url_for('index', i=session['i'], json='1'))
    #    else:
    #        return redirect(url_for('index', i=session['i']))
    if request.args.get('from_login', False) or (re.search(r'user/(register|sign-in)', str(request.referrer)) and 'next=' not in str(request.referrer)):
        next_page = app.user_manager.make_safe_url_function(request.args.get('next', page_after_login()))
        if next_page is None:
            logmessage("Invalid page " + str(next_page))
            next_page = 'interview_list'
        if next_page not in ('interview_list', 'interviews'):
            return redirect(get_url_from_file_reference(next_page))
    if daconfig.get('session list interview', None) is not None:
        if is_json:
            return redirect(url_for('index', i=daconfig.get('session list interview'), from_list='1', json='1'))
        else:
            return redirect(url_for('index', i=daconfig.get('session list interview'), from_list='1'))
    exclude_invalid = not bool(current_user.has_role('admin', 'developer'))
    resume_interview = request.args.get('resume', None)
    if resume_interview is None and daconfig.get('auto resume interview', None) is not None and (request.args.get('from_login', False) or (re.search(r'user/(register|sign-in)', str(request.referrer)) and 'next=' not in str(request.referrer))):
        resume_interview = daconfig['auto resume interview']
    if resume_interview is not None:
        (interviews, start_id) = user_interviews(user_id=current_user.id, secret=secret, exclude_invalid=True, filename=resume_interview, include_dict=True)
        if len(interviews) > 0:
            return redirect(url_for('index', i=interviews[0]['filename'], session=interviews[0]['session'], from_list='1'))
        return redirect(url_for('index', i=resume_interview, from_list='1'))
    next_id_code = request.args.get('next_id', None)
    if next_id_code:
        try:
            start_id = int(from_safeid(next_id_code))
            assert start_id >= 0
            show_back = True
        except:
            start_id = None
            show_back = False
    else:
        start_id = None
        show_back = False
    result = user_interviews(user_id=current_user.id, secret=secret, exclude_invalid=exclude_invalid, tag=tag, start_id=start_id)
    if result is None:
        raise Exception("interview_list: could not obtain list of interviews")
    (interviews, start_id) = result
    if start_id is None:
        next_id = None
    else:
        next_id = safeid(str(start_id))
    if is_json:
        for interview in interviews:
            if 'dict' in interview:
                del interview['dict']
            if 'tags' in interview:
                interview['tags'] = sorted(interview['tags'])
        return jsonify(action="interviews", interviews=interviews, next_id=next_id)
    script = """
    <script>
      $("#deleteall").on('click', function(event){
        if (confirm(""" + json.dumps(word("Are you sure you want to delete all saved interviews?")) + """)){
          return true;
        }
        event.preventDefault();
        return false;
      });
    </script>"""
    script += global_js
    if re.search(r'user/register', str(request.referrer)) and len(interviews) == 1:
        return redirect(url_for('index', i=interviews[0]['filename'], session=interviews[0]['session'], from_list=1))
    tags_used = set()
    for interview in interviews:
        for the_tag in interview['tags']:
            if the_tag != tag:
                tags_used.add(the_tag)
    #interview_page_title = word(daconfig.get('interview page title', 'Interviews'))
    #title = word(daconfig.get('interview page heading', 'Resume an interview'))
    argu = dict(version_warning=version_warning, tags_used=sorted(tags_used) if len(tags_used) > 0 else None, numinterviews=len([y for y in interviews if not y['metadata'].get('hidden', False)]), interviews=sorted(interviews, key=valid_date_key), tag=tag, next_id=next_id, show_back=show_back) # extra_css=Markup(global_css), extra_js=Markup(script), tab_title=interview_page_title, page_title=interview_page_title, title=title
    if 'interview page template' in daconfig and daconfig['interview page template']:
        the_page = docassemble.base.functions.package_template_filename(daconfig['interview page template'])
        if the_page is None:
            raise DAError("Could not find start page template " + daconfig['start page template'])
        with open(the_page, 'r', encoding='utf-8') as fp:
            template_string = fp.read()
            response = make_response(render_template_string(template_string, **argu), 200)
            response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
            return response
    else:
        response = make_response(render_template('pages/interviews.html', **argu), 200)
        response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
        return response

def valid_date_key(x):
    if x['dict']['_internal']['starttime'] is None:
        return datetime.datetime.now()
    return x['dict']['_internal']['starttime']

def fix_secret(user=None, to_convert=None):
    #logmessage("fix_secret starting")
    if user is None:
        user = current_user
    password = str(request.form.get('password', request.form.get('new_password', None)))
    if password is not None:
        secret = str(request.cookies.get('secret', None))
        newsecret = pad_to_16(MD5Hash(data=password).hexdigest())
        if secret == 'None' or secret != newsecret:
            #logmessage("fix_secret: calling substitute_secret with " + str(secret) + ' and ' + str(newsecret))
            #logmessage("fix_secret: setting newsecret session")
            session['newsecret'] = substitute_secret(str(secret), newsecret, user=user, to_convert=to_convert)
        # else:
        #     logmessage("fix_secret: secrets are the same")
    else:
        logmessage("fix_secret: password not in request")

def login_or_register(sender, user, source, **extra):
    #logmessage("login or register!")
    if 'i' in session: #TEMPORARY
        get_session(session['i'])
    to_convert = []
    if 'tempuser' in session:
        to_convert.extend(sub_temp_user_dict_key(session['tempuser'], user.id))
    if 'sessions' in session:
        for filename, info in session['sessions'].items():
            if (filename, info['uid']) not in to_convert:
                to_convert.append((filename, info['uid']))
                save_user_dict_key(info['uid'], filename, priors=True, user=user)
                update_session(filename, key_logged=True)
    fix_secret(user=user, to_convert=to_convert)
    sub_temp_other(user)
    if not (source == 'register' and daconfig.get('confirm registration', False)):
        session['user_id'] = user.id
    if user.language:
        session['language'] = user.language
        docassemble.base.functions.set_language(user.language)

def update_last_login(user):
    user.last_login = datetime.datetime.utcnow()
    db.session.commit()

@user_logged_in.connect_via(app)
def _on_user_login(sender, user, **extra):
    #logmessage("on user login")
    update_last_login(user)
    login_or_register(sender, user, 'login', **extra)
    #flash(word('You have signed in successfully.'), 'success')

@user_changed_password.connect_via(app)
def _on_password_change(sender, user, **extra):
    #logmessage("on password change")
    fix_secret(user=user)

# @user_reset_password.connect_via(app)
# def _on_password_reset(sender, user, **extra):
#     #logmessage("on password reset")
#     fix_secret(user=user)

@user_registered.connect_via(app)
def on_register_hook(sender, user, **extra):
    #why did I not just import it globally?
    #from docassemble.webapp.users.models import Role
    user_invite = extra.get('user_invite', None)
    this_user_role = None
    if user_invite is not None:
        this_user_role = db.session.execute(select(Role).filter_by(id=user_invite.role_id)).scalar()
    if this_user_role is None:
        this_user_role = db.session.execute(select(Role).filter_by(name='user')).scalar()
    roles_to_remove = []
    for role in user.roles:
        roles_to_remove.append(role)
    for role in roles_to_remove:
        user.roles.remove(role)
    user.roles.append(this_user_role)
    db.session.commit()
    update_last_login(user)
    login_or_register(sender, user, 'register', **extra)

@app.route("/fax_callback", methods=['POST'])
@csrf.exempt
def fax_callback():
    if twilio_config is None:
        logmessage("fax_callback: Twilio not enabled")
        return ('', 204)
    post_data = request.form.copy()
    if 'FaxSid' not in post_data or 'AccountSid' not in post_data:
        logmessage("fax_callback: FaxSid and/or AccountSid missing")
        return ('', 204)
    tconfig = None
    for config_name, config_info in twilio_config['name'].items():
        if 'account sid' in config_info and config_info['account sid'] == post_data['AccountSid']:
            tconfig = config_info
    if tconfig is None:
        logmessage("fax_callback: account sid of fax callback did not match any account sid in the Twilio configuration")
    if 'fax' not in tconfig or tconfig['fax'] in (False, None):
        logmessage("fax_callback: fax feature not enabled")
        return ('', 204)
    params = {}
    for param in ('FaxSid', 'From', 'To', 'RemoteStationId', 'FaxStatus', 'ApiVersion', 'OriginalMediaUrl', 'NumPages', 'MediaUrl', 'ErrorCode', 'ErrorMessage'):
        params[param] = post_data.get(param, None)
    the_key = 'da:faxcallback:sid:' + post_data['FaxSid']
    pipe = r.pipeline()
    pipe.set(the_key, json.dumps(params))
    pipe.expire(the_key, 86400)
    pipe.execute()
    return ('', 204)

@app.route("/clicksend_fax_callback", methods=['POST'])
@csrf.exempt
def clicksend_fax_callback():
    if clicksend_config is None:
        logmessage("clicksend_fax_callback: Clicksend not enabled")
        return ('', 204)
    post_data = request.form.copy()
    if 'message_id' not in post_data:
        logmessage("clicksend_fax_callback: message_id missing")
        return ('', 204)
    the_key = 'da:faxcallback:sid:' + post_data['message_id']
    the_json = r.get(the_key)
    try:
        params = json.loads(the_json)
    except:
        logmessage("clicksend_fax_callback: message_id not found")
        return ('', 204)
    for param in ('timestamp_send', 'timestamp', 'message_id', 'status', 'status_code', 'status_text', 'error_code', 'error_text', 'custom_string', 'user_id', 'subaccount_id', 'message_type'):
        params[param] = post_data.get(param, None)
    pipe = r.pipeline()
    pipe.set(the_key, json.dumps(params))
    pipe.expire(the_key, 86400)
    pipe.execute()
    return ('', 204)

@app.route("/voice", methods=['POST', 'GET'])
@csrf.exempt
def voice():
    docassemble.base.functions.set_language(DEFAULT_LANGUAGE)
    resp = twilio.twiml.voice_response.VoiceResponse()
    if twilio_config is None:
        logmessage("voice: ignoring call to voice because Twilio not enabled")
        return Response(str(resp), mimetype='text/xml')
    if 'voice' not in twilio_config['name']['default'] or twilio_config['name']['default']['voice'] in (False, None):
        logmessage("voice: ignoring call to voice because voice feature not enabled")
        return Response(str(resp), mimetype='text/xml')
    if "AccountSid" not in request.form or request.form["AccountSid"] != twilio_config['name']['default'].get('account sid', None):
        logmessage("voice: request to voice did not authenticate")
        return Response(str(resp), mimetype='text/xml')
    for item in request.form:
        logmessage("voice: item " + str(item) + " is " + str(request.form[item]))
    with resp.gather(action=url_for("digits_endpoint"), finishOnKey='#', method="POST", timeout=10, numDigits=5) as gg:
        gg.say(word("Please enter the four digit code, followed by the pound sign."))

    # twilio_config = daconfig.get('twilio', None)
    # if twilio_config is None:
    #     logmessage("Could not get twilio configuration")
    #     return
    # twilio_caller_id = twilio_config.get('number', None)
    # if "To" in request.form and request.form["To"] != '':
    #     dial = resp.dial(callerId=twilio_caller_id)
    #     if phone_pattern.match(request.form["To"]):
    #         dial.number(request.form["To"])
    #     else:
    #         dial.client(request.form["To"])
    # else:
    #     resp.say("Thanks for calling!")

    return Response(str(resp), mimetype='text/xml')

@app.route("/digits", methods=['POST', 'GET'])
@csrf.exempt
def digits_endpoint():
    docassemble.base.functions.set_language(DEFAULT_LANGUAGE)
    resp = twilio.twiml.voice_response.VoiceResponse()
    if twilio_config is None:
        logmessage("digits: ignoring call to digits because Twilio not enabled")
        return Response(str(resp), mimetype='text/xml')
    if "AccountSid" not in request.form or request.form["AccountSid"] != twilio_config['name']['default'].get('account sid', None):
        logmessage("digits: request to digits did not authenticate")
        return Response(str(resp), mimetype='text/xml')
    if "Digits" in request.form:
        the_digits = re.sub(r'[^0-9]', '', request.form["Digits"])
        logmessage("digits: got " + str(the_digits))
        phone_number = r.get('da:callforward:' + str(the_digits))
        if phone_number is None:
            resp.say(word("I am sorry.  The code you entered is invalid or expired.  Goodbye."))
            resp.hangup()
        else:
            phone_number = phone_number.decode()
            dial = resp.dial(number=phone_number)
            r.delete('da:callforward:' + str(the_digits))
    else:
        logmessage("digits: no digits received")
        resp.say(word("No access code was entered."))
        resp.hangup()
    return Response(str(resp), mimetype='text/xml')

def sms_body(phone_number, body='question', config='default'):
    if twilio_config is None:
        raise DAError("sms_body: Twilio not enabled")
    if config not in twilio_config['name']:
        raise DAError("sms_body: specified config value, " + str(config) + ", not in Twilio configuration")
    tconfig = twilio_config['name'][config]
    if 'sms' not in tconfig or tconfig['sms'] in (False, None, 0):
        raise DAError("sms_body: sms feature is not enabled in Twilio configuration")
    if 'account sid' not in tconfig:
        raise DAError("sms_body: account sid not in Twilio configuration")
    if 'number' not in tconfig:
        raise DAError("sms_body: phone number not in Twilio configuration")
    if 'doing_sms' in session:
        raise DAError("Cannot call sms_body from within sms_body")
    form = dict(To=tconfig['number'], From=phone_number, Body=body, AccountSid=tconfig['account sid'])
    base_url = url_for('rootindex', _external=True)
    url_root = base_url
    tbackup = docassemble.base.functions.backup_thread_variables()
    sbackup = backup_session()
    session['doing_sms'] = True
    resp = do_sms(form, base_url, url_root, save=False)
    restore_session(sbackup)
    docassemble.base.functions.restore_thread_variables(tbackup)
    if resp is None or len(resp.verbs) == 0 or len(resp.verbs[0].verbs) == 0:
        return None
    return resp.verbs[0].verbs[0].body

def favicon_file(filename):
    the_dir = docassemble.base.functions.package_data_filename(daconfig.get('favicon', 'docassemble.webapp:data/static/favicon'))
    if the_dir is None or not os.path.isdir(the_dir):
        logmessage("favicon_file: could not find favicon directory")
        return ('File not found', 404)
    the_file = os.path.join(the_dir, filename)
    if not os.path.isfile(the_file):
        return ('File not found', 404)
    extension, mimetype = get_ext_and_mimetype(the_file)
    response = send_file(the_file, mimetype=mimetype)
    return response

@app.route("/favicon.ico", methods=['GET'])
def favicon():
    return favicon_file('favicon.ico')
@app.route("/apple-touch-icon.png", methods=['GET'])
def apple_touch_icon():
    return favicon_file('apple-touch-icon.png')
@app.route("/favicon-32x32.png", methods=['GET'])
def favicon_md():
    return favicon_file('favicon-32x32.png')
@app.route("/favicon-16x16.png", methods=['GET'])
def favicon_sm():
    return favicon_file('favicon-16x16.png')
@app.route("/manifest.json", methods=['GET'])
def favicon_manifest_json():
    return favicon_file('manifest.json')
@app.route("/safari-pinned-tab.svg", methods=['GET'])
def favicon_safari_pinned_tab():
    return favicon_file('safari-pinned-tab.svg')
@app.route("/android-chrome-192x192.png", methods=['GET'])
def favicon_android_md():
    return favicon_file('android-chrome-192x192.png')
@app.route("/android-chrome-512x512.png", methods=['GET'])
def favicon_android_lg():
    return favicon_file('android-chrome-512x512.png')
@app.route("/mstile-150x150.png", methods=['GET'])
def favicon_mstile():
    return favicon_file('mstile-150x150.png')
@app.route("/browserconfig.xml", methods=['GET'])
def favicon_browserconfig():
    return favicon_file('browserconfig.xml')

@app.route("/robots.txt", methods=['GET'])
def robots():
    if 'robots' not in daconfig and daconfig.get('allow robots', False):
        response = make_response("User-agent: *\nDisallow:", 200)
        response.mimetype = "text/plain"
        return response
    the_file = docassemble.base.functions.package_data_filename(daconfig.get('robots', 'docassemble.webapp:data/static/robots.txt'))
    if the_file is None:
        return ('File not found', 404)
    if not os.path.isfile(the_file):
        return ('File not found', 404)
    response = send_file(the_file, mimetype='text/plain')
    return response

@app.route("/sms", methods=['POST'])
@csrf.exempt
def sms():
    #logmessage("Received: " + str(request.form))
    form = request.form
    base_url = url_for('rootindex', _external=True)
    url_root = base_url
    resp = do_sms(form, base_url, url_root)
    return Response(str(resp), mimetype='text/xml')

def do_sms(form, base_url, url_root, config='default', save=True):
    docassemble.base.functions.set_language(DEFAULT_LANGUAGE)
    resp = twilio.twiml.messaging_response.MessagingResponse()
    special_messages = []
    if twilio_config is None:
        logmessage("do_sms: ignoring message to sms because Twilio not enabled")
        return resp
    if "AccountSid" not in form or form["AccountSid"] not in twilio_config['account sid']:
        logmessage("do_sms: request to sms did not authenticate")
        return resp
    if "To" not in form or form["To"] not in twilio_config['number']:
        logmessage("do_sms: request to sms ignored because recipient number " + str(form.get('To', None)) + " not in configuration, " + str(twilio_config))
        return resp
    tconfig = twilio_config['number'][form["To"]]
    if 'sms' not in tconfig or tconfig['sms'] in (False, None, 0):
        logmessage("do_sms: ignoring message to sms because SMS not enabled")
        return resp
    if "From" not in form or not re.search(r'[0-9]', form["From"]):
        logmessage("do_sms: request to sms ignored because unable to determine caller ID")
        return resp
    if "Body" not in form:
        logmessage("do_sms: request to sms ignored because message had no content")
        return resp
    inp = form['Body'].strip()
    #logmessage("do_sms: received >" + inp + "<")
    key = 'da:sms:client:' + form["From"] + ':server:' + tconfig['number']
    for try_num in (0, 1):
        sess_contents = r.get(key)
        if sess_contents is None:
            #logmessage("do_sms: received input '" + str(inp) + "' from new user")
            yaml_filename = tconfig.get('default interview', default_yaml_filename)
            if 'dispatch' in tconfig and isinstance(tconfig['dispatch'], dict):
                if inp.lower() in tconfig['dispatch']:
                    yaml_filename = tconfig['dispatch'][inp.lower()]
                    #logmessage("do_sms: using interview from dispatch: " + str(yaml_filename))
            if yaml_filename is None:
                #logmessage("do_sms: request to sms ignored because no interview could be determined")
                return resp
            if (not DEBUG) and (yaml_filename.startswith('docassemble.base') or yaml_filename.startswith('docassemble.demo')):
                raise Exception("do_sms: not authorized to run interviews in docassemble.base or docassemble.demo")
            secret = random_string(16)
            uid = get_unique_name(yaml_filename, secret)
            new_temp_user = TempUser()
            db.session.add(new_temp_user)
            db.session.commit()
            sess_info = dict(yaml_filename=yaml_filename, uid=uid, secret=secret, number=form["From"], encrypted=True, tempuser=new_temp_user.id, user_id=None, session_uid=random_string(10))
            r.set(key, pickle.dumps(sess_info))
            accepting_input = False
        else:
            try:
                sess_info = fix_pickle_obj(sess_contents)
            except:
                logmessage("do_sms: unable to decode session information")
                return resp
            accepting_input = True
        if 'session_uid' not in sess_info:
            sess_info['session_uid'] = random_string(10)
        if inp.lower() in (word('exit'), word('quit')):
            logmessage("do_sms: exiting")
            if save:
                reset_user_dict(sess_info['uid'], sess_info['yaml_filename'], temp_user_id=sess_info['tempuser'])
            r.delete(key)
            return resp
        obtain_lock(sess_info['uid'], sess_info['yaml_filename'])
        steps, user_dict, is_encrypted = fetch_user_dict(sess_info['uid'], sess_info['yaml_filename'], secret=sess_info['secret'])
        if user_dict is None:
            r.delete(key)
            continue
        break
    encrypted = sess_info['encrypted']
    action = None
    action_performed = False
    while True:
        if user_dict.get('multi_user', False) is True and encrypted is True:
            encrypted = False
            update_session(sess_info['yaml_filename'], encrypted=encrypted, uid=sess_info['uid'])
            is_encrypted = encrypted
            r.set(key, pickle.dumps(sess_info))
            if save:
                decrypt_session(sess_info['secret'], user_code=sess_info['uid'], filename=sess_info['yaml_filename'])
        if user_dict.get('multi_user', False) is False and encrypted is False:
            encrypted = True
            update_session(sess_info['yaml_filename'], encrypted=encrypted, uid=sess_info['uid'])
            is_encrypted = encrypted
            r.set(key, pickle.dumps(sess_info))
            if save:
                encrypt_session(sess_info['secret'], user_code=sess_info['uid'], filename=sess_info['yaml_filename'])
        interview = docassemble.base.interview_cache.get_interview(sess_info['yaml_filename'])
        if 'skip' not in user_dict['_internal']:
            user_dict['_internal']['skip'] = {}
        # if 'smsgather' in user_dict['_internal']:
        #     #logmessage("do_sms: need to gather smsgather " + user_dict['_internal']['smsgather'])
        #     sms_variable = user_dict['_internal']['smsgather']
        # else:
        #     sms_variable = None
        # if action is not None:
        #     action_manual = True
        # else:
        #     action_manual = False
        user = None
        if sess_info['user_id'] is not None:
            user = load_user(sess_info['user_id'])
        if user is None:
            ci = dict(user=dict(is_anonymous=True, is_authenticated=False, email=None, theid=sess_info['tempuser'], the_user_id='t' + str(sess_info['tempuser']), roles=['user'], firstname='SMS', lastname='User', nickname=None, country=None, subdivisionfirst=None, subdivisionsecond=None, subdivisionthird=None, organization=None, timezone=None, location=None, session_uid=sess_info['session_uid'], device_id=form["From"]), session=sess_info['uid'], secret=sess_info['secret'], yaml_filename=sess_info['yaml_filename'], interface='sms', url=base_url, url_root=url_root, encrypted=encrypted, headers={}, clientip=None, method=None, skip=user_dict['_internal']['skip'], sms_sender=form["From"])
        else:
            ci = dict(user=dict(is_anonymous=False, is_authenticated=True, email=user.email, theid=user.id, the_user_id=user.id, roles=user.roles, firstname=user.first_name, lastname=user.last_name, nickname=user.nickname, country=user.country, subdivisionfirst=user.subdivisionfirst, subdivisionsecond=user.subdivisionsecond, subdivisionthird=user.subdivisionthird, organization=user.organization, timezone=user.timezone, location=None, session_uid=sess_info['session_uid'], device_id=form["From"]), session=sess_info['uid'], secret=sess_info['secret'], yaml_filename=sess_info['yaml_filename'], interface='sms', url=base_url, url_root=url_root, encrypted=encrypted, headers={}, clientip=None, method=None, skip=user_dict['_internal']['skip'])
        if action is not None:
            logmessage("do_sms: setting action to " + str(action))
            ci.update(action)
        interview_status = docassemble.base.parse.InterviewStatus(current_info=ci)
        interview.assemble(user_dict, interview_status)
        logmessage("do_sms: back from assemble 1; had been seeking variable " + str(interview_status.sought))
        logmessage("do_sms: question is " + interview_status.question.name)
        if action is not None:
            logmessage('do_sms: question is now ' + interview_status.question.name + ' because action')
            sess_info['question'] = interview_status.question.name
            r.set(key, pickle.dumps(sess_info))
        elif 'question' in sess_info and sess_info['question'] != interview_status.question.name:
            if inp not in (word('?'), word('back'), word('question'), word('exit')):
                logmessage("do_sms: blanking the input because question changed from " + str(sess_info['question']) + " to " + str(interview_status.question.name))
                sess_info['question'] = interview_status.question.name
                inp = 'question'
                r.set(key, pickle.dumps(sess_info))

        #logmessage("do_sms: inp is " + inp.lower() + " and steps is " + str(steps) + " and can go back is " + str(interview_status.can_go_back))
        m = re.search(r'^(' + word('menu') + '|' + word('link') + ')([0-9]+)', inp.lower())
        if m:
            #logmessage("Got " + inp)
            arguments = {}
            selection_type = m.group(1)
            selection_number = int(m.group(2)) - 1
            links = []
            menu_items = []
            sms_info = as_sms(interview_status, user_dict, links=links, menu_items=menu_items)
            target_url = None
            if selection_type == word('menu') and selection_number < len(menu_items):
                (target_url, label) = menu_items[selection_number]
            if selection_type == word('link') and selection_number < len(links):
                (target_url, label) = links[selection_number]
            if target_url is not None:
                uri_params = re.sub(r'^[\?]*\?', r'', target_url)
                for statement in re.split(r'&', uri_params):
                    parts = re.split(r'=', statement)
                    arguments[parts[0]] = parts[1]
            if 'action' in arguments:
                #logmessage(myb64unquote(urllibunquote(arguments['action'])))
                action = json.loads(myb64unquote(urllibunquote(arguments['action'])))
                #logmessage("Action is " + str(action))
                action_performed = True
                accepting_input = False
                inp = ''
                continue
            break
        if inp.lower() == word('back'):
            if 'skip' in user_dict['_internal'] and len(user_dict['_internal']['skip']):
                max_entry = -1
                for the_entry in user_dict['_internal']['skip'].keys():
                    if the_entry > max_entry:
                        max_entry = the_entry
                if max_entry in user_dict['_internal']['skip']:
                    del user_dict['_internal']['skip'][max_entry]
                if 'command_cache' in user_dict['_internal'] and max_entry in user_dict['_internal']['command_cache']:
                    del user_dict['_internal']['command_cache'][max_entry]
                save_user_dict(sess_info['uid'], user_dict, sess_info['yaml_filename'], secret=sess_info['secret'], encrypt=encrypted, changed=False, steps=steps)
                accepting_input = False
                inp = ''
                continue
            if steps > 1 and interview_status.can_go_back:
                old_user_dict = user_dict
                steps, user_dict, is_encrypted = fetch_previous_user_dict(sess_info['uid'], sess_info['yaml_filename'], secret=sess_info['secret'])
                if 'question' in sess_info:
                    del sess_info['question']
                    r.set(key, pickle.dumps(sess_info))
                accepting_input = False
                inp = ''
                continue
            break
        break
    false_list = [word('no'), word('n'), word('false'), word('f')]
    true_list = [word('yes'), word('y'), word('true'), word('t')]
    inp_lower = inp.lower()
    skip_it = False
    changed = False
    nothing_more = False
    if accepting_input:
        if inp_lower == word('?'):
            sms_info = as_sms(interview_status, user_dict)
            message = ''
            if sms_info['help'] is None:
                message += word('Sorry, no help is available for this question.')
            else:
                message += sms_info['help']
            message += "\n" + word("To read the question again, type question.")
            resp.message(message)
            release_lock(sess_info['uid'], sess_info['yaml_filename'])
            return resp
        if inp_lower == word('question'):
            accepting_input = False
    user_entered_skip = bool(inp_lower == word('skip'))
    if accepting_input:
        saveas = None
        uses_util = False
        uncheck_others = False
        if len(interview_status.question.fields) > 0:
            question = interview_status.question
            if question.question_type == "fields":
                field = None
                next_field = None
                for the_field in interview_status.get_field_list():
                    if hasattr(the_field, 'datatype') and the_field.datatype in ('html', 'note', 'script', 'css'):
                        continue
                    if interview_status.is_empty_mc(the_field):
                        continue
                    if the_field.number in user_dict['_internal']['skip']:
                        continue
                    if field is None:
                        field = the_field
                    elif next_field is None:
                        next_field = the_field
                    else:
                        break
                if field is None:
                    logmessage("do_sms: unclear what field is necessary!")
                    # if 'smsgather' in user_dict['_internal']:
                    #     del user_dict['_internal']['smsgather']
                    field = interview_status.question.fields[0]
                    next_field = None
                saveas = myb64unquote(field.saveas)
            else:
                if hasattr(interview_status.question.fields[0], 'saveas'):
                    saveas = myb64unquote(interview_status.question.fields[0].saveas)
                    logmessage("do_sms: variable to set is " + str(saveas))
                else:
                    saveas = None
                field = interview_status.question.fields[0]
                next_field = None
            if question.question_type == "settrue":
                if inp_lower == word('ok'):
                    data = 'True'
                else:
                    data = None
            elif question.question_type == 'signature':
                filename = 'canvas.png'
                extension = 'png'
                mimetype = 'image/png'
                temp_image_file = tempfile.NamedTemporaryFile(suffix='.' + extension)
                image = Image.new("RGBA", (200, 50))
                image.save(temp_image_file.name, 'PNG')
                (file_number, extension, mimetype) = save_numbered_file(filename, temp_image_file.name, yaml_file_name=sess_info['yaml_filename'], uid=sess_info['uid'])
                saveas_tr = sub_indices(saveas, user_dict)
                if inp_lower == word('x'):
                    the_string = saveas + " = docassemble.base.util.DAFile('" + saveas_tr + "', filename='" + str(filename) + "', number=" + str(file_number) + ", mimetype='" + str(mimetype) + "', extension='" + str(extension) + "')"
                    try:
                        exec('import docassemble.base.util', user_dict)
                        exec(the_string, user_dict)
                        if not changed:
                            steps += 1
                            user_dict['_internal']['steps'] = steps
                            changed = True
                    except Exception as errMess:
                        logmessage("do_sms: error: " + str(errMess))
                        special_messages.append(word("Error") + ": " + str(errMess))
                    skip_it = True
                    data = repr('')
                else:
                    data = None
            elif hasattr(field, 'datatype') and field.datatype in ("ml", "mlarea"):
                try:
                    exec("import docassemble.base.util", user_dict)
                except Exception as errMess:
                    special_messages.append("Error: " + str(errMess))
                if 'ml_train' in interview_status.extras and field.number in interview_status.extras['ml_train']:
                    if not interview_status.extras['ml_train'][field.number]:
                        use_for_training = 'False'
                    else:
                        use_for_training = 'True'
                else:
                    use_for_training = 'True'
                if 'ml_group' in interview_status.extras and field.number in interview_status.extras['ml_group']:
                    data = 'docassemble.base.util.DAModel(' + repr(saveas) + ', group_id=' + repr(interview_status.extras['ml_group'][field.number]) + ', text=' + repr(inp) + ', store=' + repr(interview.get_ml_store()) + ', use_for_training=' + use_for_training + ')'
                else:
                    data = 'docassemble.base.util.DAModel(' + repr(saveas) + ', text=' + repr(inp) + ', store=' + repr(interview.get_ml_store()) + ', use_for_training=' + use_for_training + ')'
            elif hasattr(field, 'datatype') and field.datatype in ("file", "files", "camera", "user", "environment", "camcorder", "microphone"):
                if user_entered_skip and not interview_status.extras['required'][field.number]:
                    skip_it = True
                    data = repr('')
                else:
                    files_to_process = []
                    num_media = int(form.get('NumMedia', '0'))
                    fileindex = 0
                    while True:
                        if field.datatype == "file" and fileindex > 0:
                            break
                        if fileindex >= num_media or 'MediaUrl' + str(fileindex) not in form:
                            break
                        #logmessage("mime type is" + form.get('MediaContentType' + str(fileindex), 'Unknown'))
                        mimetype = form.get('MediaContentType' + str(fileindex), 'image/jpeg')
                        extension = re.sub(r'\.', r'', mimetypes.guess_extension(mimetype))
                        if extension == 'jpe':
                            extension = 'jpg'
                        # original_extension = extension
                        # if extension == 'gif':
                        #     extension == 'png'
                        #     mimetype = 'image/png'
                        filename = 'file' + '.' + extension
                        file_number = get_new_file_number(sess_info['uid'], filename, yaml_file_name=sess_info['yaml_filename'])
                        saved_file = SavedFile(file_number, extension=extension, fix=True, should_not_exist=True)
                        the_url = form['MediaUrl' + str(fileindex)]
                        #logmessage("Fetching from >" + the_url + "<")
                        saved_file.fetch_url(the_url)
                        process_file(saved_file, saved_file.path, mimetype, extension)
                        files_to_process.append((filename, file_number, mimetype, extension))
                        fileindex += 1
                    if len(files_to_process) > 0:
                        elements = []
                        indexno = 0
                        saveas_tr = sub_indices(saveas, user_dict)
                        for (filename, file_number, mimetype, extension) in files_to_process:
                            elements.append("docassemble.base.util.DAFile(" + repr(saveas_tr + "[" + str(indexno) + "]") + ", filename=" + repr(filename) + ", number=" + str(file_number) + ", mimetype=" + repr(mimetype) + ", extension=" + repr(extension) + ")")
                            indexno += 1
                        the_string = saveas + " = docassemble.base.util.DAFileList(" + repr(saveas_tr) + ", elements=[" + ", ".join(elements) + "])"
                        try:
                            exec('import docassemble.base.util', user_dict)
                            exec(the_string, user_dict)
                            if not changed:
                                steps += 1
                                user_dict['_internal']['steps'] = steps
                                changed = True
                        except Exception as errMess:
                            logmessage("do_sms: error: " + str(errMess))
                            special_messages.append(word("Error") + ": " + str(errMess))
                        skip_it = True
                        data = repr('')
                    else:
                        data = None
                        if interview_status.extras['required'][field.number]:
                            special_messages.append(word("You must attach a file."))
            elif question.question_type == "yesno" or (hasattr(field, 'datatype') and (hasattr(field, 'datatype') and field.datatype == 'boolean' and (hasattr(field, 'sign') and field.sign > 0))):
                if inp_lower in true_list:
                    data = 'True'
                    if question.question_type == "fields" and hasattr(field, 'uncheckothers') and field.uncheckothers is not False:
                        uncheck_others = field
                elif inp_lower in false_list:
                    data = 'False'
                else:
                    data = None
            elif question.question_type == "yesnomaybe" or (hasattr(field, 'datatype') and (field.datatype == 'threestate' and (hasattr(field, 'sign') and field.sign > 0))):
                if inp_lower in true_list:
                    data = 'True'
                    if question.question_type == "fields" and hasattr(field, 'uncheckothers') and field.uncheckothers is not False:
                        uncheck_others = field
                elif inp_lower in false_list:
                    data = 'False'
                else:
                    data = 'None'
            elif question.question_type == "noyes" or (hasattr(field, 'datatype') and (field.datatype in ('noyes', 'noyeswide') or (field.datatype == 'boolean' and (hasattr(field, 'sign') and field.sign < 0)))):
                if inp_lower in true_list:
                    data = 'False'
                elif inp_lower in false_list:
                    data = 'True'
                    if question.question_type == "fields" and hasattr(field, 'uncheckothers') and field.uncheckothers is not False:
                        uncheck_others = field
                else:
                    data = None
            elif question.question_type in ('noyesmaybe', 'noyesmaybe', 'noyeswidemaybe') or (hasattr(field, 'datatype') and field.datatype == 'threestate' and (hasattr(field, 'sign') and field.sign < 0)):
                if inp_lower in true_list:
                    data = 'False'
                elif inp_lower in false_list:
                    data = 'True'
                    if question.question_type == "fields" and hasattr(field, 'uncheckothers') and field.uncheckothers is not False:
                        uncheck_others = field
                else:
                    data = 'None'
            elif question.question_type == 'multiple_choice' or hasattr(field, 'choicetype') or (hasattr(field, 'datatype') and field.datatype in ('object', 'object_radio', 'multiselect', 'object_multiselect', 'checkboxes', 'object_checkboxes')) or (hasattr(field, 'inputtype') and field.inputtype == 'radio'):
                cdata, choice_list = get_choices_with_abb(interview_status, field, user_dict)
                data = None
                if hasattr(field, 'datatype') and field.datatype in ('multiselect', 'object_multiselect', 'checkboxes', 'object_checkboxes') and saveas is not None:
                    if 'command_cache' not in user_dict['_internal']:
                        user_dict['_internal']['command_cache'] = {}
                    if field.number not in user_dict['_internal']['command_cache']:
                        user_dict['_internal']['command_cache'][field.number] = []
                    docassemble.base.parse.ensure_object_exists(saveas, field.datatype, user_dict, commands=user_dict['_internal']['command_cache'][field.number])
                    saveas = saveas + '.gathered'
                    data = 'True'
                if (user_entered_skip or (inp_lower == word('none') and hasattr(field, 'datatype') and field.datatype in ('multiselect', 'object_multiselect', 'checkboxes', 'object_checkboxes'))) and ((hasattr(field, 'disableothers') and field.disableothers) or (hasattr(field, 'datatype') and field.datatype in ('multiselect', 'object_multiselect', 'checkboxes', 'object_checkboxes')) or not (interview_status.extras['required'][field.number] or (question.question_type == 'multiple_choice' and hasattr(field, 'saveas')))):
                    logmessage("do_sms: skip accepted")
                    # user typed 'skip,' or, where checkboxes, 'none.'  Also:
                    # field is skippable, either because it has disableothers, or it is a checkbox field, or
                    # it is not required.  Multiple choice fields with saveas are considered required.
                    if hasattr(field, 'datatype'):
                        if field.datatype in ('object', 'object_radio'):
                            skip_it = True
                            data = repr('')
                        if field.datatype in ('multiselect', 'object_multiselect', 'checkboxes', 'object_checkboxes'):
                            for choice in choice_list:
                                if choice[1] is None:
                                    continue
                                user_dict['_internal']['command_cache'][field.number].append(choice[1] + ' = False')
                        elif (question.question_type == 'multiple_choice' and hasattr(field, 'saveas')) or hasattr(field, 'choicetype'):
                            if user_entered_skip:
                                skip_it = True
                                data = repr('')
                            else:
                                logmessage("do_sms: setting skip_it to True")
                                skip_it = True
                                data = repr('')
                        elif field.datatype == 'integer':
                            data = '0'
                        elif field.datatype in ('number', 'float', 'currency', 'range'):
                            data = '0.0'
                        else:
                            data = repr('')
                    else:
                        data = repr('')
                else:
                    # There is a real value here
                    if hasattr(field, 'datatype') and field.datatype in ('object_multiselect', 'object_checkboxes'):
                        true_values = set()
                        for selection in re.split(r' *[,;] *', inp_lower):
                            for potential_abb, value in cdata['abblower'].items():
                                if selection and selection.startswith(potential_abb):
                                    for choice in choice_list:
                                        if value == choice[0]:
                                            true_values.add(choice[2])
                        the_saveas = myb64unquote(field.saveas)
                        logmessage("do_sms: the_saveas is " + repr(the_saveas))
                        for choice in choice_list:
                            if choice[2] is None:
                                continue
                            if choice[2] in true_values:
                                logmessage("do_sms: " + choice[2] + " is in true_values")
                                the_string = 'if ' + choice[2] + ' not in ' + the_saveas + '.elements:\n    ' + the_saveas + '.append(' + choice[2] + ')'
                            else:
                                the_string = 'if ' + choice[2] + ' in ' + the_saveas + '.elements:\n    ' + the_saveas + '.remove(' + choice[2] + ')'
                            user_dict['_internal']['command_cache'][field.number].append(the_string)
                    elif hasattr(field, 'datatype') and field.datatype in ('multiselect', 'checkboxes'):
                        true_values = set()
                        for selection in re.split(r' *[,;] *', inp_lower):
                            for potential_abb, value in cdata['abblower'].items():
                                if selection and selection.startswith(potential_abb):
                                    for choice in choice_list:
                                        if value == choice[0]:
                                            true_values.add(choice[1])
                        for choice in choice_list:
                            if choice[1] is None:
                                continue
                            if choice[1] in true_values:
                                the_string = choice[1] + ' = True'
                            else:
                                the_string = choice[1] + ' = False'
                            user_dict['_internal']['command_cache'][field.number].append(the_string)
                    else:
                        #regular multiple choice
                        #logmessage("do_sms: user selected " + inp_lower + " and data is " + str(cdata))
                        for potential_abb, value in cdata['abblower'].items():
                            if inp_lower.startswith(potential_abb):
                                #logmessage("do_sms: user selected " + value)
                                for choice in choice_list:
                                    #logmessage("do_sms: considering " + choice[0])
                                    if value == choice[0]:
                                        #logmessage("do_sms: found a match")
                                        saveas = choice[1]
                                        if hasattr(field, 'datatype') and field.datatype in ('object', 'object_radio'):
                                            data = choice[2]
                                        else:
                                            data = repr(choice[2])
                                        break
                                break
            elif hasattr(field, 'datatype') and field.datatype == 'integer':
                if user_entered_skip and not interview_status.extras['required'][field.number]:
                    data = repr('')
                    skip_it = True
                else:
                    data = re.sub(r'[^0-9\-\.]', '', inp)
                    if data == '':
                        data = '0'
                    try:
                        the_value = eval("int(" + repr(data) + ")")
                        data = "int(" + repr(data) + ")"
                    except:
                        special_messages.append('"' + inp + '" ' + word("is not a whole number."))
                        data = None
            elif hasattr(field, 'datatype') and field.datatype in ('date', 'datetime'):
                if user_entered_skip and not interview_status.extras['required'][field.number]:
                    data = repr('')
                    skip_it = True
                else:
                    try:
                        dateutil.parser.parse(inp)
                        data = "docassemble.base.util.as_datetime(" + repr(inp) + ")"
                        uses_util = True
                    except Exception as the_err:
                        logmessage("do_sms: date validation error was " + str(the_err))
                        if field.datatype == 'date':
                            special_messages.append('"' + inp + '" ' + word("is not a valid date."))
                        else:
                            special_messages.append('"' + inp + '" ' + word("is not a valid date and time."))
                        data = None
            elif hasattr(field, 'datatype') and field.datatype == 'time':
                if user_entered_skip and not interview_status.extras['required'][field.number]:
                    data = repr('')
                    skip_it = True
                else:
                    try:
                        dateutil.parser.parse(inp)
                        data = "docassemble.base.util.as_datetime(" + repr(inp) + ").time()"
                        uses_util = True
                    except Exception as the_err:
                        logmessage("do_sms: time validation error was " + str(the_err))
                        special_messages.append('"' + inp + '" ' + word("is not a valid time."))
                        data = None
            elif hasattr(field, 'datatype') and field.datatype == 'range':
                if user_entered_skip and not interview_status.extras['required'][field.number]:
                    data = repr('')
                    skip_it = True
                else:
                    data = re.sub(r'[^0-9\-\.]', '', inp)
                    try:
                        the_value = eval("float(" + repr(data) + ")", user_dict)
                        if the_value > int(interview_status.extras['max'][field.number]) or the_value < int(interview_status.extras['min'][field.number]):
                            special_messages.append('"' + inp + '" ' + word("is not within the range."))
                            data = None
                    except:
                        data = None
            elif hasattr(field, 'datatype') and field.datatype in ('number', 'float', 'currency'):
                if user_entered_skip and not interview_status.extras['required'][field.number]:
                    data = repr('')
                    skip_it = True
                else:
                    data = re.sub(r'[^0-9\-\.]', '', inp)
                    if data == '':
                        data = '0.0'
                    try:
                        the_value = eval("float(" + json.dumps(data) + ")", user_dict)
                        data = "float(" + json.dumps(data) + ")"
                    except:
                        special_messages.append('"' + inp + '" ' + word("is not a valid number."))
                        data = None
            else:
                if user_entered_skip:
                    if interview_status.extras['required'][field.number]:
                        data = repr(inp)
                    else:
                        data = repr('')
                        skip_it = True
                else:
                    data = repr(inp)
        else:
            data = None
        if data is None:
            logmessage("do_sms: could not process input: " + inp)
            special_messages.append(word("I do not understand what you mean by") + ' "' + inp + '."')
        else:
            if uses_util:
                exec("import docassemble.base.util", user_dict)
            if uncheck_others:
                for other_field in interview_status.get_field_list():
                    if hasattr(other_field, 'datatype') and other_field.datatype == 'boolean' and other_field is not uncheck_others and 'command_cache' in user_dict['_internal'] and other_field.number in user_dict['_internal']['command_cache']:
                        for command_index in range(len(user_dict['_internal']['command_cache'][other_field.number])):
                            if other_field.sign > 0:
                                user_dict['_internal']['command_cache'][other_field.number][command_index] = re.sub(r'= True$', '= False', user_dict['_internal']['command_cache'][other_field.number][command_index])
                            else:
                                user_dict['_internal']['command_cache'][other_field.number][command_index] = re.sub(r'= False$', '= True', user_dict['_internal']['command_cache'][other_field.number][command_index])
            the_string = saveas + ' = ' + data
            try:
                if not skip_it:
                    if hasattr(field, 'disableothers') and field.disableothers and hasattr(field, 'saveas'):
                        logmessage("do_sms: disabling others")
                        next_field = None
                    if next_field is not None:
                        if 'command_cache' not in user_dict['_internal']:
                            user_dict['_internal']['command_cache'] = {}
                        if field.number not in user_dict['_internal']['command_cache']:
                            user_dict['_internal']['command_cache'][field.number] = []
                        user_dict['_internal']['command_cache'][field.number].append(the_string)
                        logmessage("do_sms: storing in command cache: " + str(the_string))
                    else:
                        for the_field in interview_status.get_field_list():
                            if interview_status.is_empty_mc(the_field):
                                logmessage("do_sms: a field is empty")
                                the_saveas = myb64unquote(the_field.saveas)
                                if 'command_cache' not in user_dict['_internal']:
                                    user_dict['_internal']['command_cache'] = {}
                                if the_field.number not in user_dict['_internal']['command_cache']:
                                    user_dict['_internal']['command_cache'][the_field.number] = []
                                if hasattr(the_field, 'datatype'):
                                    if the_field.datatype in ('object_multiselect', 'object_checkboxes'):
                                        docassemble.base.parse.ensure_object_exists(the_saveas, the_field.datatype, user_dict, commands=user_dict['_internal']['command_cache'][the_field.number])
                                        user_dict['_internal']['command_cache'][the_field.number].append(the_saveas + '.clear()')
                                        user_dict['_internal']['command_cache'][the_field.number].append(the_saveas + '.gathered = True')
                                    elif the_field.datatype in ('object', 'object_radio'):
                                        try:
                                            eval(the_saveas, user_dict)
                                        except:
                                            user_dict['_internal']['command_cache'][the_field.number].append(the_saveas + ' = None')
                                    elif the_field.datatype in ('multiselect', 'checkboxes'):
                                        docassemble.base.parse.ensure_object_exists(the_saveas, the_field.datatype, user_dict, commands=user_dict['_internal']['command_cache'][the_field.number])
                                        user_dict['_internal']['command_cache'][the_field.number].append(the_saveas + '.gathered = True')
                                    else:
                                        user_dict['_internal']['command_cache'][the_field.number].append(the_saveas + ' = None')
                                else:
                                    user_dict['_internal']['command_cache'][the_field.number].append(the_saveas + ' = None')
                        if 'command_cache' in user_dict['_internal']:
                            for field_num in sorted(user_dict['_internal']['command_cache'].keys()):
                                for pre_string in user_dict['_internal']['command_cache'][field_num]:
                                    logmessage("do_sms: doing command cache: " + pre_string)
                                    exec(pre_string, user_dict)
                        logmessage("do_sms: doing regular: " + the_string)
                        exec(the_string, user_dict)
                        if not changed:
                            steps += 1
                            user_dict['_internal']['steps'] = steps
                            changed = True
                if next_field is None:
                    if skip_it:
                        # Run the commands that we have been storing up
                        if 'command_cache' in user_dict['_internal']:
                            for field_num in sorted(user_dict['_internal']['command_cache'].keys()):
                                for pre_string in user_dict['_internal']['command_cache'][field_num]:
                                    logmessage("do_sms: doing command cache: " + pre_string)
                                    exec(pre_string, user_dict)
                            if not changed:
                                steps += 1
                                user_dict['_internal']['steps'] = steps
                                changed = True
                    logmessage("do_sms: next_field is None")
                    if 'skip' in user_dict['_internal']:
                        user_dict['_internal']['skip'].clear()
                    if 'command_cache' in user_dict['_internal']:
                        user_dict['_internal']['command_cache'].clear()
                    # if 'sms_variable' in interview_status.current_info:
                    #     del interview_status.current_info['sms_variable']
                else:
                    logmessage("do_sms: next_field is not None")
                    user_dict['_internal']['skip'][field.number] = True
                    #user_dict['_internal']['smsgather'] = interview_status.sought
                # if 'smsgather' in user_dict['_internal'] and user_dict['_internal']['smsgather'] == saveas:
                #     #logmessage("do_sms: deleting " + user_dict['_internal']['smsgather'])
                #     del user_dict['_internal']['smsgather']
            except Exception as the_err:
                logmessage("do_sms: failure to set variable with " + the_string)
                logmessage("do_sms: error was " + str(the_err))
                release_lock(sess_info['uid'], sess_info['yaml_filename'])
                #if 'uid' in session:
                #    del session['uid']
                return resp
        if changed and next_field is None and question.name not in user_dict['_internal']['answers']:
            logmessage("do_sms: setting internal answers for " + str(question.name))
            question.mark_as_answered(user_dict)
        interview.assemble(user_dict, interview_status)
        logmessage("do_sms: back from assemble 2; had been seeking variable " + str(interview_status.sought))
        logmessage("do_sms: question is now " + str(interview_status.question.name))
        sess_info['question'] = interview_status.question.name
        r.set(key, pickle.dumps(sess_info))
    else:
        logmessage("do_sms: not accepting input.")
    if interview_status.question.question_type in ("restart", "exit", "logout", "exit_logout", "new_session"):
        logmessage("do_sms: exiting because of restart or exit")
        if save:
            obtain_lock(sess_info['uid'], sess_info['yaml_filename'])
            reset_user_dict(sess_info['uid'], sess_info['yaml_filename'], temp_user_id=sess_info['tempuser'])
            release_lock(sess_info['uid'], sess_info['yaml_filename'])
        r.delete(key)
        if interview_status.question.question_type in ('restart', 'new_session'):
            sess_info = dict(yaml_filename=sess_info['yaml_filename'], uid=get_unique_name(sess_info['yaml_filename'], sess_info['secret']), secret=sess_info['secret'], number=form["From"], encrypted=True, tempuser=sess_info['tempuser'], user_id=None)
            r.set(key, pickle.dumps(sess_info))
            form = dict(To=form['To'], From=form['From'], AccountSid=form['AccountSid'], Body=word('question'))
            return do_sms(form, base_url, url_root, config=config, save=True)
    else:
        if not interview_status.can_go_back:
            user_dict['_internal']['steps_offset'] = steps
        #I had commented this out in do_sms(), but not in index()
        #user_dict['_internal']['answers'] = {}
        #Why do this?
        if (not interview_status.followed_mc) and len(user_dict['_internal']['answers']):
            user_dict['_internal']['answers'].clear()
        # if interview_status.question.name and interview_status.question.name in user_dict['_internal']['answers']:
        #     del user_dict['_internal']['answers'][interview_status.question.name]
        #logmessage("do_sms: " + as_sms(interview_status, user_dict))
        #twilio_client = TwilioRestClient(tconfig['account sid'], tconfig['auth token'])
        #message = twilio_client.messages.create(to=form["From"], from_=form["To"], body=as_sms(interview_status, user_dict))
        logmessage("do_sms: calling as_sms")
        sms_info = as_sms(interview_status, user_dict)
        qoutput = sms_info['question']
        if sms_info['next'] is not None:
            logmessage("do_sms: next variable is " + sms_info['next'])
            if interview_status.sought is None:
                logmessage("do_sms: sought variable is None")
            #user_dict['_internal']['smsgather'] = interview_status.sought
        if (accepting_input or changed or action_performed or sms_info['next'] is not None) and save:
            save_user_dict(sess_info['uid'], user_dict, sess_info['yaml_filename'], secret=sess_info['secret'], encrypt=encrypted, changed=changed, steps=steps)
        for special_message in special_messages:
            qoutput = re.sub(r'XXXXMESSAGE_AREAXXXX', "\n" + special_message + 'XXXXMESSAGE_AREAXXXX', qoutput)
        qoutput = re.sub(r'XXXXMESSAGE_AREAXXXX', '', qoutput)
        if user_dict.get('multi_user', False) is True and encrypted is True:
            encrypted = False
            update_session(sess_info['yaml_filename'], encrypted=encrypted, uid=sess_info['uid'])
            is_encrypted = encrypted
            r.set(key, pickle.dumps(sess_info))
            if save:
                decrypt_session(sess_info['secret'], user_code=sess_info['uid'], filename=sess_info['yaml_filename'])
        if user_dict.get('multi_user', False) is False and encrypted is False:
            encrypted = True
            update_session(sess_info['yaml_filename'], encrypted=encrypted, uid=sess_info['uid'])
            is_encrypted = encrypted
            r.set(key, pickle.dumps(sess_info))
            if save:
                encrypt_session(sess_info['secret'], user_code=sess_info['uid'], filename=sess_info['yaml_filename'])
        if len(interview_status.attachments) > 0:
            with resp.message(qoutput) as m:
                media_count = 0
                for attachment in interview_status.attachments:
                    if media_count >= 9:
                        break
                    for doc_format in attachment['formats_to_use']:
                        if media_count >= 9:
                            break
                        if doc_format not in ('pdf', 'rtf'):
                            continue
                        filename = attachment['filename'] + '.' + docassemble.base.parse.extension_of_doc_format[doc_format]
                        #saved_file = save_numbered_file(filename, attachment['file'][doc_format], yaml_file_name=sess_info['yaml_filename'], uid=sess_info['uid'])
                        url = url_for('serve_stored_file', _external=True, uid=sess_info['uid'], number=attachment['file'][doc_format], filename=attachment['filename'], extension=docassemble.base.parse.extension_of_doc_format[doc_format])
                        #logmessage('sms: url is ' + str(url))
                        m.media(url)
                        media_count += 1
        else:
            resp.message(qoutput)
    release_lock(sess_info['uid'], sess_info['yaml_filename'])
    #logmessage(str(form))
    return resp

def get_api_key():
    api_key = request.args.get('key', None)
    if api_key is None and request.method in ('POST', 'PUT', 'PATCH'):
        post_data = request.get_json(silent=True)
        if post_data is None:
            post_data = request.form.copy()
        if 'key' in post_data:
            api_key = post_data['key']
    if api_key is None and 'X-API-Key' in request.cookies:
        api_key = request.cookies['X-API-Key']
    if api_key is None and 'X-API-Key' in request.headers:
        api_key = request.headers['X-API-Key']
    return api_key

def api_verify(req, roles=None):
    api_key = get_api_key()
    if api_key is None:
        logmessage("api_verify: no API key provided")
        return False
    api_key = encrypt_api_key(api_key)
    rkeys = r.keys('da:apikey:userid:*:key:' + api_key + ':info')
    if len(rkeys) == 0:
        logmessage("api_verify: API key not found")
        return False
    try:
        info = json.loads(r.get(rkeys[0].decode()).decode())
    except:
        logmessage("api_verify: API information could not be unpacked")
        return False
    m = re.match(r'da:apikey:userid:([0-9]+):key:' + re.escape(api_key) + ':info', rkeys[0].decode())
    if not m:
        logmessage("api_verify: user id could not be extracted")
        return False
    user_id = m.group(1)
    if not isinstance(info, dict):
        logmessage("api_verify: API information was in the wrong format")
        return False
    if len(info['constraints']) > 0:
        clientip = get_requester_ip(request)
        if info['method'] == 'ip' and clientip not in info['constraints']:
            logmessage("api_verify: IP address " + str(clientip) + " did not match")
            return False
        if info['method'] == 'referer':
            if not request.referrer:
                the_referer = request.headers.get('Origin', None)
                if not the_referer:
                    logmessage("api_verify: could not authorize based on referer because no referer provided")
                    return False
            else:
                the_referer = request.referrer
            matched = False
            for constraint in info['constraints']:
                constraint = re.sub(r'^[\*]+|[\*]+$', '', constraint)
                constraint = re.escape(constraint)
                constraint = re.sub(r'\\\*+', '.*', constraint)
                the_referer = re.sub(r'\?.*', '', the_referer)
                the_referer = re.sub(r'^https?://([^/]*)/', r'\1', the_referer)
                if re.search(constraint, the_referer):
                    matched = True
                    break
            if not matched:
                logmessage("api_verify: authorization failure referer " + str(the_referer) + " could not be matched")
                return False
    user = db.session.execute(select(UserModel).options(db.joinedload(UserModel.roles)).where(UserModel.id == user_id)).scalar()
    if user is None or user.social_id.startswith('disabled$'):
        logmessage("api_verify: user does not exist")
        return False
    if not user.active:
        logmessage("api_verify: user is no longer active")
        return False
    login_user(user, remember=False)
    update_last_login(user)
    if roles:
        ok_role = False
        for role in roles:
            if current_user.has_role(role):
                ok_role = True
                break
        if not ok_role:
            logmessage("api_verify: user did not have correct privileges for resource")
            return False
    docassemble.base.functions.this_thread.current_info = current_info(req=request, interface='api', device_id=request.cookies.get('ds', None), session_uid=current_user.email)
    return True

def jsonify_with_status(data, code):
    resp = jsonify(data)
    resp.status_code = code
    return resp

def true_or_false(text):
    if text in (False, None) or text == 0 or str(text).lower().strip() in ('0', 'false', 'f'):
        return False
    return True

def get_user_list(include_inactive=False, start_id=None):
    if not (current_user.is_authenticated and current_user.has_role('admin', 'advocate')):
        raise Exception("You cannot call get_user_list() unless you are an administrator or advocate")
    user_length = 0
    user_list = []
    while True:
        there_are_more = False
        filter_list = []
        if start_id is not None:
            filter_list.append(UserModel.id > start_id)
        if not include_inactive:
            filter_list.append(UserModel.active == True)
        the_users = select(UserModel).options(db.joinedload(UserModel.roles))
        if len(filter_list) > 0:
            the_users = the_users.where(*filter_list)
        the_users = the_users.order_by(UserModel.id).limit(PAGINATION_LIMIT_PLUS_ONE)
        results_in_query = 0
        for user in db.session.execute(the_users).unique().scalars():
            results_in_query += 1
            if results_in_query == PAGINATION_LIMIT_PLUS_ONE:
                there_are_more = True
                break
            start_id = user.id
            if user.social_id.startswith('disabled$'):
                continue
            if user_length == PAGINATION_LIMIT:
                there_are_more = True
                break
            user_info = {}
            user_info['privileges'] = []
            for role in user.roles:
                user_info['privileges'].append(role.name)
            for attrib in ('id', 'email', 'first_name', 'last_name', 'country', 'subdivisionfirst', 'subdivisionsecond', 'subdivisionthird', 'organization', 'timezone', 'language'):
                user_info[attrib] = getattr(user, attrib)
            if include_inactive:
                user_info['active'] = getattr(user, 'active')
            user_list.append(user_info)
            user_length += 1
        if user_length == PAGINATION_LIMIT or results_in_query < PAGINATION_LIMIT_PLUS_ONE:
            break
    if not there_are_more:
        start_id = None
    return (user_list, start_id)

@app.route('/translation_file', methods=['POST'])
@login_required
@roles_required(['admin', 'developer'])
def translation_file():
    setup_translation()
    form = Utilities(request.form)
    yaml_filename = form.interview.data
    if yaml_filename is None or not re.search(r'\S', yaml_filename):
        flash(word("You must provide an interview filename"), 'error')
        return redirect(url_for('utilities'))
    tr_lang = form.language.data
    if tr_lang is None or not re.search(r'\S', tr_lang):
        flash(word("You must provide a language"), 'error')
        return redirect(url_for('utilities'))
    try:
        interview_source = docassemble.base.parse.interview_source_from_string(yaml_filename)
    except DAError:
        flash(word("Invalid interview"), 'error')
        return redirect(url_for('utilities'))
    interview_source.update()
    interview_source.translating = True
    interview = interview_source.get_interview()
    tr_cache = {}
    if len(interview.translations) > 0:
        for item in interview.translations:
            if item.lower().endswith(".xlsx"):
                the_xlsx_file = docassemble.base.functions.package_data_filename(item)
                if not os.path.isfile(the_xlsx_file):
                    continue
                df = pandas.read_excel(the_xlsx_file, na_values=['NaN', '-NaN', '#NA', '#N/A'], keep_default_na=False)
                invalid = False
                for column_name in ('interview', 'question_id', 'index_num', 'hash', 'orig_lang', 'tr_lang', 'orig_text', 'tr_text'):
                    if column_name not in df.columns:
                        invalid = True
                        break
                if invalid:
                    continue
                for indexno in df.index:
                    try:
                        assert df['interview'][indexno]
                        assert df['question_id'][indexno]
                        assert df['index_num'][indexno] >= 0
                        assert df['hash'][indexno]
                        assert df['orig_lang'][indexno]
                        assert df['tr_lang'][indexno]
                        assert df['orig_text'][indexno] != ''
                        assert df['tr_text'][indexno] != ''
                        if isinstance(df['orig_text'][indexno], float):
                            assert not math.isnan(df['orig_text'][indexno])
                        if isinstance(df['tr_text'][indexno], float):
                            assert not math.isnan(df['tr_text'][indexno])
                    except:
                        continue
                    the_dict = {'interview': str(df['interview'][indexno]), 'question_id': str(df['question_id'][indexno]), 'index_num': df['index_num'][indexno], 'hash': str(df['hash'][indexno]), 'orig_lang': str(df['orig_lang'][indexno]), 'tr_lang': str(df['tr_lang'][indexno]), 'orig_text': str(df['orig_text'][indexno]), 'tr_text': str(df['tr_text'][indexno])}
                    if df['orig_text'][indexno] not in tr_cache:
                        tr_cache[df['orig_text'][indexno]] = {}
                    if df['orig_lang'][indexno] not in tr_cache[df['orig_text'][indexno]]:
                        tr_cache[df['orig_text'][indexno]][df['orig_lang'][indexno]] = {}
                    tr_cache[df['orig_text'][indexno]][df['orig_lang'][indexno]][df['tr_lang'][indexno]] = the_dict
            elif item.lower().endswith(".xlf") or item.lower().endswith(".xliff"):
                the_xlf_file = docassemble.base.functions.package_data_filename(item)
                if not os.path.isfile(the_xlf_file):
                    continue
                tree = ET.parse(the_xlf_file)
                root = tree.getroot()
                indexno = 1
                if root.attrib['version'] == "1.2":
                    for the_file in root.iter('{urn:oasis:names:tc:xliff:document:1.2}file'):
                        source_lang = the_file.attrib.get('source-language', 'en')
                        target_lang = the_file.attrib.get('target-language', 'en')
                        source_filename = the_file.attrib.get('original', yaml_filename)
                        for transunit in the_file.iter('{urn:oasis:names:tc:xliff:document:1.2}trans-unit'):
                            orig_text = ''
                            tr_text = ''
                            for source in transunit.iter('{urn:oasis:names:tc:xliff:document:1.2}source'):
                                if source.text:
                                    orig_text += source.text
                                for mrk in source:
                                    orig_text += mrk.text
                                    if mrk.tail:
                                        orig_text += mrk.tail
                            for target in transunit.iter('{urn:oasis:names:tc:xliff:document:1.2}target'):
                                if target.text:
                                    tr_text += target.text
                                for mrk in target:
                                    tr_text += mrk.text
                                    if mrk.tail:
                                        tr_text += mrk.tail
                            if orig_text == '' or tr_text == '':
                                continue
                            the_dict = {'interview': source_filename, 'question_id': 'Unknown' + str(indexno), 'index_num': transunit.attrib.get('id', str(indexno)), 'hash': hashlib.md5(orig_text.encode('utf-8')).hexdigest(), 'orig_lang': source_lang, 'tr_lang': target_lang, 'orig_text': orig_text, 'tr_text': tr_text}
                            if orig_text not in tr_cache:
                                tr_cache[orig_text] = {}
                            if source_lang not in tr_cache[orig_text]:
                                tr_cache[orig_text][source_lang] = {}
                            tr_cache[orig_text][source_lang][target_lang] = the_dict
                            indexno += 1
                elif root.attrib['version'] == "2.0":
                    source_lang = root.attrib['srcLang']
                    target_lang = root.attrib['trgLang']
                    for the_file in root.iter('{urn:oasis:names:tc:xliff:document:2.0}file'):
                        source_filename = the_file.attrib.get('original', yaml_filename)
                        for unit in the_file.iter('{urn:oasis:names:tc:xliff:document:2.0}unit'):
                            question_id = unit.attrib.get('id', 'Unknown' + str(indexno))
                            for segment in unit.iter('{urn:oasis:names:tc:xliff:document:2.0}segment'):
                                orig_text = ''
                                tr_text = ''
                                for source in transunit.iter('{urn:oasis:names:tc:xliff:document:2.0}source'):
                                    if source.text:
                                        orig_text += source.text
                                    for mrk in source:
                                        orig_text += mrk.text
                                        if mrk.tail:
                                            orig_text += mrk.tail
                                for target in transunit.iter('{urn:oasis:names:tc:xliff:document:2.0}target'):
                                    if target.text:
                                        tr_text += target.text
                                    for mrk in target:
                                        tr_text += mrk.text
                                        if mrk.tail:
                                            tr_text += mrk.tail
                                if orig_text == '' or tr_text == '':
                                    continue
                                the_dict = {'interview': source_filename, 'question_id': question_id, 'index_num': segment.attrib.get('id', str(indexno)), 'hash': hashlib.md5(orig_text.encode('utf-8')).hexdigest(), 'orig_lang': source_lang, 'tr_lang': target_lang, 'orig_text': orig_text, 'tr_text': tr_text}
                                if orig_text not in tr_cache:
                                    tr_cache[orig_text] = {}
                                if source_lang not in tr_cache[orig_text]:
                                    tr_cache[orig_text][source_lang] = {}
                                tr_cache[orig_text][source_lang][target_lang] = the_dict
                                indexno += 1
    if form.filetype.data == 'XLSX':
        temp_file = tempfile.NamedTemporaryFile(suffix='.xlsx', delete=False)
        xlsx_filename = docassemble.base.functions.space_to_underscore(os.path.splitext(os.path.basename(re.sub(r'.*:', '', yaml_filename)))[0]) + "_" + tr_lang + ".xlsx"
        workbook = xlsxwriter.Workbook(temp_file.name)
        worksheet = workbook.add_worksheet()
        bold = workbook.add_format({'bold': 1})
        text = workbook.add_format()
        text.set_align('top')
        fixedcell = workbook.add_format()
        fixedcell.set_align('top')
        fixedcell.set_text_wrap()
        fixedunlockedcell = workbook.add_format()
        fixedunlockedcell.set_align('top')
        fixedunlockedcell.set_text_wrap()
        fixedunlockedcell.set_locked(False)
        fixed = workbook.add_format()
        fixedone = workbook.add_format()
        fixedone.set_bold()
        fixedone.set_font_color('green')
        fixedtwo = workbook.add_format()
        fixedtwo.set_bold()
        fixedtwo.set_font_color('blue')
        fixedunlocked = workbook.add_format()
        fixedunlockedone = workbook.add_format()
        fixedunlockedone.set_bold()
        fixedunlockedone.set_font_color('green')
        fixedunlockedtwo = workbook.add_format()
        fixedunlockedtwo.set_bold()
        fixedunlockedtwo.set_font_color('blue')
        wholefixed = workbook.add_format()
        wholefixed.set_align('top')
        wholefixed.set_text_wrap()
        wholefixedone = workbook.add_format()
        wholefixedone.set_bold()
        wholefixedone.set_font_color('green')
        wholefixedone.set_align('top')
        wholefixedone.set_text_wrap()
        wholefixedtwo = workbook.add_format()
        wholefixedtwo.set_bold()
        wholefixedtwo.set_font_color('blue')
        wholefixedtwo.set_align('top')
        wholefixedtwo.set_text_wrap()
        wholefixedunlocked = workbook.add_format()
        wholefixedunlocked.set_align('top')
        wholefixedunlocked.set_text_wrap()
        wholefixedunlocked.set_locked(False)
        wholefixedunlockedone = workbook.add_format()
        wholefixedunlockedone.set_bold()
        wholefixedunlockedone.set_font_color('green')
        wholefixedunlockedone.set_align('top')
        wholefixedunlockedone.set_text_wrap()
        wholefixedunlockedone.set_locked(False)
        wholefixedunlockedtwo = workbook.add_format()
        wholefixedunlockedtwo.set_bold()
        wholefixedunlockedtwo.set_font_color('blue')
        wholefixedunlockedtwo.set_align('top')
        wholefixedunlockedtwo.set_text_wrap()
        wholefixedunlockedtwo.set_locked(False)
        numb = workbook.add_format()
        numb.set_align('top')
        worksheet.write('A1', 'interview', bold)
        worksheet.write('B1', 'question_id', bold)
        worksheet.write('C1', 'index_num', bold)
        worksheet.write('D1', 'hash', bold)
        worksheet.write('E1', 'orig_lang', bold)
        worksheet.write('F1', 'tr_lang', bold)
        worksheet.write('G1', 'orig_text', bold)
        worksheet.write('H1', 'tr_text', bold)
        options = {
            'objects':               False,
            'scenarios':             False,
            'format_cells':          False,
            'format_columns':        False,
            'format_rows':           False,
            'insert_columns':        False,
            'insert_rows':           True,
            'insert_hyperlinks':     False,
            'delete_columns':        False,
            'delete_rows':           True,
            'select_locked_cells':   True,
            'sort':                  True,
            'autofilter':            True,
            'pivot_tables':          False,
            'select_unlocked_cells': True,
        }
        worksheet.protect('', options)
        worksheet.set_column(0, 0, 25)
        worksheet.set_column(1, 1, 15)
        worksheet.set_column(2, 2, 12)
        worksheet.set_column(6, 6, 75)
        worksheet.set_column(6, 7, 75)
        row = 1
        seen = []
        for question in interview.all_questions:
            if not hasattr(question, 'translations'):
                continue
            language = question.language
            if language == '*':
                language = interview_source.language
            if language == '*':
                language = DEFAULT_LANGUAGE
            if language == tr_lang:
                continue
            indexno = 0
            if hasattr(question, 'id'):
                question_id = question.id
            else:
                question_id = question.name
            for item in question.translations:
                if item in seen:
                    continue
                if item in tr_cache and language in tr_cache[item] and tr_lang in tr_cache[item][language]:
                    tr_text = str(tr_cache[item][language][tr_lang]['tr_text'])
                else:
                    tr_text = ''
                worksheet.write_string(row, 0, question.from_source.get_name(), text)
                worksheet.write_string(row, 1, question_id, text)
                worksheet.write_number(row, 2, indexno, numb)
                worksheet.write_string(row, 3, hashlib.md5(item.encode('utf-8')).hexdigest(), text)
                worksheet.write_string(row, 4, language, text)
                worksheet.write_string(row, 5, tr_lang, text)
                mako = mako_parts(item)
                if len(mako) == 0:
                    worksheet.write_string(row, 6, '', wholefixed)
                elif len(mako) == 1:
                    if mako[0][1] == 0:
                        worksheet.write_string(row, 6, item, wholefixed)
                    elif mako[0][1] == 1:
                        worksheet.write_string(row, 6, item, wholefixedone)
                    elif mako[0][1] == 2:
                        worksheet.write_string(row, 6, item, wholefixedtwo)
                else:
                    parts = [row, 6]
                    for part in mako:
                        if part[1] == 0:
                            parts.extend([fixed, part[0]])
                        elif part[1] == 1:
                            parts.extend([fixedone, part[0]])
                        elif part[1] == 2:
                            parts.extend([fixedtwo, part[0]])
                    parts.append(fixedcell)
                    worksheet.write_rich_string(*parts)
                mako = mako_parts(tr_text)
                if len(mako) == 0:
                    worksheet.write_string(row, 7, '', wholefixedunlocked)
                elif len(mako) == 1:
                    if mako[0][1] == 0:
                        worksheet.write_string(row, 7, tr_text, wholefixedunlocked)
                    elif mako[0][1] == 1:
                        worksheet.write_string(row, 7, tr_text, wholefixedunlockedone)
                    elif mako[0][1] == 2:
                        worksheet.write_string(row, 7, tr_text, wholefixedunlockedtwo)
                else:
                    parts = [row, 7]
                    for part in mako:
                        if part[1] == 0:
                            parts.extend([fixedunlocked, part[0]])
                        elif part[1] == 1:
                            parts.extend([fixedunlockedone, part[0]])
                        elif part[1] == 2:
                            parts.extend([fixedunlockedtwo, part[0]])
                    parts.append(fixedunlockedcell)
                    worksheet.write_rich_string(*parts)
                num_lines = item.count('\n')
                #if num_lines > 25:
                #    num_lines = 25
                if num_lines > 0:
                    worksheet.set_row(row, 15*(num_lines + 1))
                indexno += 1
                row += 1
                seen.append(item)
        for item in tr_cache:
            if item in seen or language not in tr_cache[item] or tr_lang not in tr_cache[item][language]:
                continue
            worksheet.write_string(row, 0, tr_cache[item][language][tr_lang]['interview'], text)
            worksheet.write_string(row, 1, tr_cache[item][language][tr_lang]['question_id'], text)
            worksheet.write_number(row, 2, 1000 + tr_cache[item][language][tr_lang]['index_num'], numb)
            worksheet.write_string(row, 3, tr_cache[item][language][tr_lang]['hash'], text)
            worksheet.write_string(row, 4, tr_cache[item][language][tr_lang]['orig_lang'], text)
            worksheet.write_string(row, 5, tr_cache[item][language][tr_lang]['tr_lang'], text)
            mako = mako_parts(tr_cache[item][language][tr_lang]['orig_text'])
            if len(mako) == 1:
                if mako[0][1] == 0:
                    worksheet.write_string(row, 6, tr_cache[item][language][tr_lang]['orig_text'], wholefixed)
                elif mako[0][1] == 1:
                    worksheet.write_string(row, 6, tr_cache[item][language][tr_lang]['orig_text'], wholefixedone)
                elif mako[0][1] == 2:
                    worksheet.write_string(row, 6, tr_cache[item][language][tr_lang]['orig_text'], wholefixedtwo)
            else:
                parts = [row, 6]
                for part in mako:
                    if part[1] == 0:
                        parts.extend([fixed, part[0]])
                    elif part[1] == 1:
                        parts.extend([fixedone, part[0]])
                    elif part[1] == 2:
                        parts.extend([fixedtwo, part[0]])
                parts.append(fixedcell)
                worksheet.write_rich_string(*parts)
            mako = mako_parts(tr_cache[item][language][tr_lang]['tr_text'])
            if len(mako) == 1:
                if mako[0][1] == 0:
                    worksheet.write_string(row, 7, tr_cache[item][language][tr_lang]['tr_text'], wholefixedunlocked)
                elif mako[0][1] == 1:
                    worksheet.write_string(row, 7, tr_cache[item][language][tr_lang]['tr_text'], wholefixedunlockedone)
                elif mako[0][1] == 2:
                    worksheet.write_string(row, 7, tr_cache[item][language][tr_lang]['tr_text'], wholefixedunlockedtwo)
            else:
                parts = [row, 7]
                for part in mako:
                    if part[1] == 0:
                        parts.extend([fixedunlocked, part[0]])
                    elif part[1] == 1:
                        parts.extend([fixedunlockedone, part[0]])
                    elif part[1] == 2:
                        parts.extend([fixedunlockedtwo, part[0]])
                parts.append(fixedunlockedcell)
                worksheet.write_rich_string(*parts)
            num_lines = tr_cache[item][language][tr_lang]['orig_text'].count('\n')
            if num_lines > 0:
                worksheet.set_row(row, 15*(num_lines + 1))
            row += 1
        workbook.close()
        response = send_file(temp_file.name, mimetype='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', as_attachment=True, attachment_filename=xlsx_filename)
        response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
        return response
    if form.filetype.data.startswith('XLIFF'):
        seen = set()
        translations = {}
        xliff_files = []
        if form.filetype.data == 'XLIFF 1.2':
            for question in interview.all_questions:
                if not hasattr(question, 'translations'):
                    continue
                language = question.language
                if language == '*':
                    language = interview_source.language
                if language == '*':
                    language = DEFAULT_LANGUAGE
                if language == tr_lang:
                    continue
                question_id = question.name
                lang_combo = (language, tr_lang)
                if lang_combo not in translations:
                    translations[lang_combo] = []
                for item in question.translations:
                    if item in seen:
                        continue
                    if item in tr_cache and language in tr_cache[item] and tr_lang in tr_cache[item][language]:
                        tr_text = str(tr_cache[item][language][tr_lang]['tr_text'])
                    else:
                        tr_text = ''
                    orig_mako = mako_parts(item)
                    tr_mako = mako_parts(tr_text)
                    translations[lang_combo].append([orig_mako, tr_mako])
                    seen.add(item)
            for lang_combo, translation_list in translations.items():
                temp_file = tempfile.NamedTemporaryFile(suffix='.xlf', delete=False)
                if len(translations) > 1:
                    xlf_filename = docassemble.base.functions.space_to_underscore(os.path.splitext(os.path.basename(re.sub(r'.*:', '', yaml_filename)))[0]) + "_" + lang_combo[0] + "_" + lang_combo[1] + ".xlf"
                else:
                    xlf_filename = docassemble.base.functions.space_to_underscore(os.path.splitext(os.path.basename(re.sub(r'.*:', '', yaml_filename)))[0]) + "_" + lang_combo[1] + ".xlf"
                xliff = ET.Element('xliff')
                xliff.set('xmlns', 'urn:oasis:names:tc:xliff:document:1.2')
                xliff.set('version', '1.2')
                indexno = 1
                the_file = ET.SubElement(xliff, 'file')
                the_file.set('id', 'f1')
                the_file.set('original', yaml_filename)
                the_file.set('xml:space', 'preserve')
                the_file.set('source-language', lang_combo[0])
                the_file.set('target-language', lang_combo[1])
                body = ET.SubElement(the_file, 'body')
                for item in translation_list:
                    transunit = ET.SubElement(body, 'trans-unit')
                    transunit.set('id', str(indexno))
                    transunit.set('xml:space', 'preserve')
                    source = ET.SubElement(transunit, 'source')
                    source.set('xml:space', 'preserve')
                    target = ET.SubElement(transunit, 'target')
                    target.set('xml:space', 'preserve')
                    last_elem = None
                    for (elem, i) in ((source, 0), (target, 1)):
                        if len(item[i]) == 0:
                            elem.text = ''
                        elif len(item[i]) == 1 and item[i][0][1] == 0:
                            elem.text = item[i][0][0]
                        else:
                            for part in item[i]:
                                if part[1] == 0:
                                    if last_elem is None:
                                        if elem.text is None:
                                            elem.text = ''
                                        elem.text += part[0]
                                    else:
                                        if last_elem.tail is None:
                                            last_elem.tail = ''
                                        last_elem.tail += part[0]
                                else:
                                    mrk = ET.SubElement(elem, 'mrk')
                                    mrk.set('xml:space', 'preserve')
                                    mrk.set('mtype', 'protected')
                                    mrk.text = part[0]
                                    last_elem = mrk
                    indexno += 1
                temp_file.write(ET.tostring(xliff))
                temp_file.close()
                xliff_files.append([temp_file, xlf_filename])
        elif form.filetype.data == 'XLIFF 2.0':
            for question in interview.all_questions:
                if not hasattr(question, 'translations'):
                    continue
                language = question.language
                if language == '*':
                    language = interview_source.language
                if language == '*':
                    language = DEFAULT_LANGUAGE
                if language == tr_lang:
                    continue
                question_id = question.name
                lang_combo = (language, tr_lang)
                if lang_combo not in translations:
                    translations[lang_combo] = {}
                filename = question.from_source.get_name()
                if filename not in translations[lang_combo]:
                    translations[lang_combo][filename] = {}
                if question_id not in translations[lang_combo][filename]:
                    translations[lang_combo][filename][question_id] = []
                for item in question.translations:
                    if item in seen:
                        continue
                    if item in tr_cache and language in tr_cache[item] and tr_lang in tr_cache[item][language]:
                        tr_text = str(tr_cache[item][language][tr_lang]['tr_text'])
                    else:
                        tr_text = ''
                    orig_mako = mako_parts(item)
                    tr_mako = mako_parts(tr_text)
                    translations[lang_combo][filename][question_id].append([orig_mako, tr_mako])
                    seen.add(item)
            for lang_combo, translations_by_filename in translations.items():
                temp_file = tempfile.NamedTemporaryFile(suffix='.xlf', delete=False)
                if len(translations) > 1:
                    xlf_filename = docassemble.base.functions.space_to_underscore(os.path.splitext(os.path.basename(re.sub(r'.*:', '', yaml_filename)))[0]) + "_" + lang_combo[0] + "_" + lang_combo[1] + ".xlf"
                else:
                    xlf_filename = docassemble.base.functions.space_to_underscore(os.path.splitext(os.path.basename(re.sub(r'.*:', '', yaml_filename)))[0]) + "_" + lang_combo[1] + ".xlf"
                xliff = ET.Element('xliff')
                xliff.set('xmlns', 'urn:oasis:names:tc:xliff:document:2.0')
                xliff.set('version', '2.0')
                xliff.set('srcLang', lang_combo[0])
                xliff.set('trgLang', lang_combo[1])
                file_index = 1
                indexno = 1
                for filename, translations_by_question in translations_by_filename.items():
                    the_file = ET.SubElement(xliff, 'file')
                    the_file.set('id', 'f' + str(file_index))
                    the_file.set('original', filename)
                    the_file.set('xml:space', 'preserve')
                    for question_id, translation_list in translations_by_question.items():
                        unit = ET.SubElement(the_file, 'unit')
                        unit.set('id', question_id)
                        for item in translation_list:
                            segment = ET.SubElement(unit, 'segment')
                            segment.set('id', str(indexno))
                            segment.set('xml:space', 'preserve')
                            source = ET.SubElement(segment, 'source')
                            source.set('xml:space', 'preserve')
                            target = ET.SubElement(segment, 'target')
                            target.set('xml:space', 'preserve')
                            last_elem = None
                            for (elem, i) in ((source, 0), (target, 1)):
                                if len(item[i]) == 0:
                                    elem.text = ''
                                elif len(item[i]) == 1 and item[i][0][1] == 0:
                                    elem.text = item[i][0][0]
                                else:
                                    for part in item[i]:
                                        if part[1] == 0:
                                            if last_elem is None:
                                                if elem.text is None:
                                                    elem.text = ''
                                                elem.text += part[0]
                                            else:
                                                if last_elem.tail is None:
                                                    last_elem.tail = ''
                                                last_elem.tail += part[0]
                                        else:
                                            mrk = ET.SubElement(elem, 'mrk')
                                            mrk.set('xml:space', 'preserve')
                                            mrk.set('translate', 'no')
                                            mrk.text = part[0]
                                            last_elem = mrk
                            indexno += 1
                    file_index += 1
                temp_file.write(ET.tostring(xliff))
                temp_file.close()
                xliff_files.append([temp_file, xlf_filename])
        else:
            flash(word("Bad file format"), 'error')
            return redirect(url_for('utilities'))
        if len(xliff_files) == 1:
            response = send_file(xliff_files[0][0].name, mimetype='application/xml', as_attachment=True, attachment_filename=xliff_files[0][1])
        else:
            zip_file = tempfile.NamedTemporaryFile(suffix='.zip', delete=False)
            zip_file_name = docassemble.base.functions.space_to_underscore(os.path.splitext(os.path.basename(re.sub(r'.*:', '', yaml_filename)))[0]) + "_" + tr_lang + ".zip"
            with zipfile.ZipFile(zip_file, mode='w') as zf:
                for item in xliff_files:
                    info = zipfile.ZipInfo(item[1])
                    with open(item[0].name, 'rb') as fp:
                        zf.writestr(info, fp.read())
                zf.close()
            response = send_file(zip_file.name, mimetype='application/xml', as_attachment=True, attachment_filename=zip_file_name)
        response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
        return response
    flash(word("Bad file format"), 'error')
    return redirect(url_for('utilities'))

@app.route('/api/user_list', methods=['GET'])
@cross_origin(origins='*', methods=['GET', 'HEAD'], automatic_options=True)
def api_user_list():
    if not api_verify(request, roles=['admin', 'advocate']):
        return jsonify_with_status("Access denied.", 403)
    include_inactive = true_or_false(request.args.get('include_inactive', False))
    next_id_code = request.args.get('next_id', None)
    if next_id_code:
        try:
            start_id = int(from_safeid(next_id_code))
            assert start_id >= 0
        except:
            start_id = None
    else:
        start_id = None
    (user_list, start_id) = get_user_list(include_inactive=include_inactive, start_id=start_id)
    if start_id is None:
        next_id = None
    else:
        next_id = safeid(str(start_id))
    return jsonify(dict(next_id=next_id, items=user_list))

def get_user_info(user_id=None, email=None, case_sensitive=False):
    if current_user.is_anonymous:
        raise Exception("You cannot call get_user_info() unless you are logged in")
    if user_id is not None:
        assert isinstance(user_id, int)
    if user_id is None and email is None:
        user_id = current_user.id
    if email is not None:
        assert isinstance(email, str)
        email = email.strip()
    if not (current_user.has_role('admin', 'advocate')):
        if (user_id is not None and current_user.id != user_id) or (email is not None and current_user.email != email):
            raise Exception("You cannot call get_user_info() unless you are an administrator or advocate")
    user_info = dict(privileges=[])
    if user_id is not None:
        user = db.session.execute(select(UserModel).options(db.joinedload(UserModel.roles)).where(UserModel.id == user_id)).scalar()
    else:
        if case_sensitive:
            user = db.session.execute(select(UserModel).options(db.joinedload(UserModel.roles)).filter_by(email=email)).scalar()
        else:
            email = re.sub(r'\%', '', email)
            user = db.session.execute(select(UserModel).options(db.joinedload(UserModel.roles)).where(UserModel.email.ilike(email))).scalar()
    if user is None or user.social_id.startswith('disabled$'):
        return None
    for role in user.roles:
        user_info['privileges'].append(role.name)
    for attrib in ('id', 'email', 'first_name', 'last_name', 'country', 'subdivisionfirst', 'subdivisionsecond', 'subdivisionthird', 'organization', 'timezone', 'language', 'active'):
        user_info[attrib] = getattr(user, attrib)
    user_info['account_type'] = re.sub(r'\$.*', '', user.social_id)
    return user_info

def make_user_inactive(user_id=None, email=None):
    if current_user.is_anonymous:
        raise Exception("You cannot call make_user_inactive() unless you are logged in")
    if not (current_user.has_role('admin')):
        raise Exception("You cannot call make_user_inactive() unless you are an administrator")
    if user_id is None and email is None:
        raise Exception("You must call make_user_inactive() with a user ID or an e-mail address")
    if user_id is not None:
        user = db.session.execute(select(UserModel).filter_by(id=user_id)).scalar()
    else:
        assert isinstance(email, str)
        email = email.strip()
        user = db.session.execute(select(UserModel).filter_by(email=email)).scalar()
    if user is None:
        raise Exception("User not found")
    user.active = False
    db.session.commit()

@app.route('/api/user', methods=['GET', 'POST'])
@csrf.exempt
@cross_origin(origins='*', methods=['GET', 'POST', 'HEAD'], automatic_options=True)
def api_user():
    if not api_verify(request):
        return jsonify_with_status("Access denied.", 403)
    try:
        user_info = get_user_info(user_id=current_user.id)
    except Exception as err:
        return jsonify_with_status("Error obtaining user information: " + str(err), 400)
    if user_info is None:
        return jsonify_with_status('User not found', 404)
    if request.method == 'GET':
        return jsonify(user_info)
    if request.method == 'POST':
        post_data = request.get_json(silent=True)
        if post_data is None:
            post_data = request.form.copy()
        info = {}
        for key in ('first_name', 'last_name', 'country', 'subdivisionfirst', 'subdivisionsecond', 'subdivisionthird', 'organization', 'timezone', 'language', 'password'):
            if key in post_data:
                info[key] = post_data[key]
        if 'password' in info and not current_user.has_role('admin'):
            return jsonify_with_status("You must have admin privileges to change a password.", 403)
        set_user_info(user_id=current_user.id, **info)
        return ('', 204)
    return ('', 204)

@app.route('/api/user/privileges', methods=['GET'])
@csrf.exempt
@cross_origin(origins='*', methods=['GET', 'HEAD'], automatic_options=True)
def api_user_privileges():
    if not api_verify(request):
        return jsonify_with_status("Access denied.", 403)
    try:
        user_info = get_user_info(user_id=current_user.id)
    except Exception as err:
        return jsonify_with_status("Error obtaining user information: " + str(err), 400)
    if user_info is None:
        return jsonify_with_status('User not found', 404)
    if request.method == 'GET':
        return jsonify(user_info['privileges'])

@app.route('/api/user/new', methods=['POST'])
@csrf.exempt
@cross_origin(origins='*', methods=['POST', 'HEAD'], automatic_options=True)
def api_create_user():
    if not api_verify(request, roles=['admin']):
        return jsonify_with_status("Access denied.", 403)
    post_data = request.get_json(silent=True)
    if post_data is None:
        post_data = request.form.copy()
    if 'email' in post_data and 'username' not in post_data: # temporary
        post_data['username'] = post_data['email'].strip()
        del post_data['email']
    if 'username' not in post_data:
        return jsonify_with_status("An e-mail address must be supplied.", 400)
    info = {}
    for key in ('first_name', 'last_name', 'country', 'subdivisionfirst', 'subdivisionsecond', 'subdivisionthird', 'organization', 'timezone', 'language'):
        if key in post_data:
            info[key] = post_data[key].strip()
    if 'privileges' in post_data and isinstance(post_data['privileges'], list):
        role_list = post_data['privileges']
    else:
        try:
            role_list = json.loads(post_data.get('privileges', '[]'))
        except:
            role_list = [post_data['privileges']]
    if not isinstance(role_list, list):
        if not isinstance(role_list, str):
            return jsonify_with_status("List of privileges must be a string or a list.", 400)
        role_list = [role_list]
    valid_role_names = set()
    for rol in db.session.execute(select(Role).where(Role.name != 'cron').order_by(Role.id)).scalars():
        valid_role_names.add(rol.name)
    for role_name in role_list:
        if role_name not in valid_role_names:
            return jsonify_with_status("Invalid privilege name.  " + role_name + " is not an existing privilege.", 400)
    password = post_data.get('password', random_alphanumeric(10)).strip()
    if len(password) < 4 or len(password) > 254:
        return jsonify_with_status("Password too short or too long", 400)
    try:
        password = str(password)
        user_id = create_user(post_data['username'], password, role_list, info)
    except Exception as err:
        return jsonify_with_status(str(err), 400)
    return jsonify_with_status(dict(user_id=user_id, password=password), 200)

@app.route('/api/user_info', methods=['GET'])
@cross_origin(origins='*', methods=['GET', 'HEAD'], automatic_options=True)
def api_user_info():
    if not api_verify(request, roles=['admin', 'advocate']):
        return jsonify_with_status("Access denied.", 403)
    if 'username' not in request.args:
        return jsonify_with_status("An e-mail address must be supplied.", 400)
    case_sensitive = true_or_false(request.args.get('case_sensitive', False))
    try:
        user_info = get_user_info(email=request.args['username'], case_sensitive=case_sensitive)
    except Exception as err:
        return jsonify_with_status("Error obtaining user information: " + str(err), 400)
    if user_info is None:
        return jsonify_with_status("User not found.", 404)
    if request.method == 'GET':
        return jsonify(user_info)

@app.route('/api/user/<int:user_id>', methods=['GET', 'DELETE', 'POST'])
@csrf.exempt
@cross_origin(origins='*', methods=['GET', 'DELETE', 'POST', 'HEAD'], automatic_options=True)
def api_user_by_id(user_id):
    if not api_verify(request):
        return jsonify_with_status("Access denied.", 403)
    try:
        user_id = int(user_id)
    except:
        return jsonify_with_status("User ID must be an integer.", 400)
    try:
        user_info = get_user_info(user_id=user_id)
    except Exception as err:
        return jsonify_with_status("Error obtaining user information: " + str(err), 400)
    if user_info is None:
        return jsonify_with_status("User not found.", 404)
    if request.method == 'GET':
        return jsonify(user_info)
    if request.method == 'DELETE':
        if user_id == 1:
            return jsonify_with_status("This user account cannot be deleted or deactivated.", 403)
        if request.args.get('remove', None) == 'account':
            if not (current_user.id == user_id or current_user.has_role('admin')):
                return jsonify_with_status("You must have admin privileges to delete user accounts.", 403)
            user_interviews(user_id=user_id, secret=None, exclude_invalid=False, action='delete_all', delete_shared=False)
            delete_user_data(user_id, r, r_user)
        elif request.args.get('remove', None) == 'account_and_shared':
            if not (current_user.id == user_id or current_user.has_role('admin')):
                return jsonify_with_status("You must have admin privileges to delete user accounts.", 403)
            user_interviews(user_id=user_id, secret=None, exclude_invalid=False, action='delete_all', delete_shared=True)
            delete_user_data(user_id, r, r_user)
        else:
            if not (current_user.id == user_id or current_user.has_role('admin')):
                return jsonify_with_status("You must have admin privileges to inactivate user accounts.", 403)
            make_user_inactive(user_id=user_id)
        return ('', 204)
    if request.method == 'POST':
        if not (current_user.has_role('admin', 'advocate') or current_user.id != user_id):
            return jsonify_with_status("You must have admin or advocate privileges to edit user information.", 403)
        post_data = request.get_json(silent=True)
        if post_data is None:
            post_data = request.form.copy()
        info = {}
        for key in ('first_name', 'last_name', 'country', 'subdivisionfirst', 'subdivisionsecond', 'subdivisionthird', 'organization', 'timezone', 'language', 'password'):
            if key in post_data:
                info[key] = post_data[key]
        if 'password' in info and not current_user.has_role('admin'):
            return jsonify_with_status("You must have admin privileges to change a password.", 403)
        try:
            set_user_info(user_id=user_id, **info)
        except Exception as err:
            return jsonify_with_status(str(err), 400)
        return ('', 204)
    return ('', 204)

@app.route('/api/fields', methods=['POST'])
@csrf.exempt
@cross_origin(origins='*', methods=['POST', 'HEAD'], automatic_options=True)
def api_fields():
    if not api_verify(request, roles=['admin', 'developer']):
        return jsonify_with_status("Access denied.", 403)
    post_data = request.get_json(silent=True)
    if post_data is None:
        post_data = request.form.copy()
    output_format = post_data.get('format', 'json')
    if output_format not in ('json', 'yaml'):
        return jsonify_with_status("Invalid output format.", 400)
    if 'template' not in request.files:
        return jsonify_with_status("File not included.", 400)
    the_files = request.files.getlist('template')
    if not the_files:
        return jsonify_with_status("File not included.", 400)
    for the_file in the_files:
        filename = secure_filename(the_file.filename)
        temp_file = tempfile.NamedTemporaryFile(prefix="datemp", delete=False)
        the_file.save(temp_file.name)
        try:
            input_format = os.path.splitext(filename.lower())[1][1:]
        except:
            input_format = 'bin'
        if input_format == 'md':
            input_format = 'markdown'
        if input_format not in ('docx', 'markdown', 'pdf'):
            return jsonify_with_status("Invalid input format.", 400)
        try:
            output = read_fields(temp_file.name, filename, input_format, output_format)
        except Exception as err:
            logmessage("api_fields: got error " + err.__class__.__name__ + ": " + str(err))
            if output_format == 'yaml':
                return jsonify_with_status("No fields could be found.", 400)
            else:
                return jsonify(dict(fields=[]))
        break
    if output_format == 'yaml':
        response = make_response(output.encode('utf-8'), '200 OK')
        response.headers['Content-type'] = 'text/plain; charset=utf-8'
    else:
        response = make_response(output.encode('utf-8'), 200)
        response.headers['Content-Type'] = 'application/json; charset=utf-8'
    return response

@app.route('/api/privileges', methods=['GET', 'DELETE', 'POST'])
@csrf.exempt
@cross_origin(origins='*', methods=['GET', 'DELETE', 'POST', 'HEAD'], automatic_options=True)
def api_privileges():
    if not api_verify(request):
        return jsonify_with_status("Access denied.", 403)
    if request.method == 'GET':
        return jsonify(get_privileges_list())
    if request.method == 'DELETE':
        if not (current_user.has_role('admin')):
            return jsonify_with_status("Access denied.", 403)
        if 'privilege' not in request.args:
            return jsonify_with_status("A privilege name must be provided.", 400)
        try:
            remove_privilege(request.args['privilege'])
        except Exception as err:
            return jsonify_with_status(str(err), 400)
        return ('', 204)
    if request.method == 'POST':
        if not (current_user.has_role('admin')):
            return jsonify_with_status("Access denied.", 403)
        post_data = request.get_json(silent=True)
        if post_data is None:
            post_data = request.form.copy()
        if 'privilege' not in post_data:
            return jsonify_with_status("A privilege name must be provided.", 400)
        try:
            add_privilege(post_data['privilege'])
        except Exception as err:
            return jsonify_with_status(str(err), 400)
        return ('', 204)

def get_privileges_list():
    if not (current_user.has_role('admin', 'developer')):
        raise Exception('You must have admin or developer privileges to call get_privileges_list().')
    role_names = []
    for role in db.session.execute(select(Role.name).order_by(Role.name)):
        role_names.append(role.name)
    return role_names

def add_privilege(privilege):
    if not (current_user.has_role('admin')):
        raise Exception('You must have admin privileges to call add_privilege().')
    role_names = get_privileges_list()
    if privilege in role_names:
        raise Exception("The given privilege already exists.")
    db.session.add(Role(name=privilege))
    db.session.commit()

def remove_privilege(privilege):
    if not (current_user.has_role('admin')):
        raise Exception('You must have admin privileges to call remove_privilege().')
    if privilege in ['user', 'admin', 'developer', 'advocate', 'cron']:
        raise Exception('The specified privilege is built-in and cannot be deleted.')
    user_role = db.session.execute(select(Role).filter_by(name='user')).scalar()
    role = db.session.execute(select(Role).filter_by(name=privilege)).scalar()
    if role is None:
        raise Exception('The privilege ' + str(privilege) + ' did not exist.')
    db.session.delete(role)
    db.session.commit()

@app.route('/api/user/<int:user_id>/privileges', methods=['GET', 'DELETE', 'POST'])
@csrf.exempt
@cross_origin(origins='*', methods=['GET', 'DELETE', 'POST', 'HEAD'], automatic_options=True)
def api_user_by_id_privileges(user_id):
    if not api_verify(request):
        return jsonify_with_status("Access denied.", 403)
    try:
        user_info = get_user_info(user_id=user_id)
    except Exception as err:
        return jsonify_with_status("Error obtaining user information: " + str(err), 400)
    if user_info is None:
        return jsonify_with_status('User not found', 404)
    if request.method == 'GET':
        return jsonify(user_info['privileges'])
    if request.method in ('DELETE', 'POST'):
        if not (current_user.has_role('admin')):
            return jsonify_with_status("Access denied.", 403)
        if request.method == 'DELETE':
            role_name = request.args.get('privilege', None)
            if role_name is None:
                return jsonify_with_status("A privilege name must be provided", 400)
            try:
                remove_user_privilege(user_id, role_name)
            except Exception as err:
                return jsonify_with_status(str(err), 400)
        elif request.method == 'POST':
            post_data = request.get_json(silent=True)
            if post_data is None:
                post_data = request.form.copy()
            role_name = post_data.get('privilege', None)
            if role_name is None:
                return jsonify_with_status("A privilege name must be provided", 400)
            try:
                add_user_privilege(user_id, role_name)
            except Exception as err:
                return jsonify_with_status(str(err), 400)
        db.session.commit()
        return ('', 204)

def add_user_privilege(user_id, privilege):
    if not (current_user.has_role('admin')):
        raise Exception('You must have admin privileges to call add_user_privilege().')
    if privilege not in get_privileges_list():
        raise Exception('The specified privilege does not exist.')
    user = db.session.execute(select(UserModel).options(db.joinedload(UserModel.roles)).where(UserModel.id == user_id)).scalar()
    if user is None or user.social_id.startswith('disabled$'):
        raise Exception("The specified user did not exist")
    for role in user.roles:
        if role.name == privilege:
            raise Exception("The user already had that privilege.")
    role_to_add = None
    for role in db.session.execute(select(Role).order_by(Role.id)).scalars():
        if role.name == privilege:
            role_to_add = role
    if role_to_add is None:
        raise Exception("The specified privilege did not exist.")
    user.roles.append(role_to_add)
    db.session.commit()

def remove_user_privilege(user_id, privilege):
    if not (current_user.has_role('admin')):
        raise Exception('You must have admin privileges to call remove_user_privilege().')
    if current_user.id == user_id and privilege == 'admin':
        raise Exception('You cannot take away the admin privilege from the current user.')
    if privilege not in get_privileges_list():
        raise Exception('The specified privilege does not exist.')
    user = db.session.execute(select(UserModel).options(db.joinedload(UserModel.roles)).where(UserModel.id == user_id)).scalar()
    if user is None or user.social_id.startswith('disabled$'):
        raise Exception("The specified user did not exist")
    role_to_remove = None
    for role in user.roles:
        if role.name == privilege:
            role_to_remove = role
    if role_to_remove is None:
        raise Exception("The user did not already have that privilege.")
    user.roles.remove(role_to_remove)
    db.session.commit()

def create_user(email, password, privileges=None, info=None):
    if current_user.is_anonymous:
        raise Exception("You cannot call create_user() unless you are logged in")
    if not (current_user.has_role('admin')):
        raise Exception("You cannot call create_user() unless you are an administrator")
    email = email.strip()
    password = str(password).strip()
    if len(password) < 4 or len(password) > 254:
        raise Exception("Password too short or too long")
    role_dict = {}
    if privileges is None:
        privileges = []
    if isinstance(privileges, DAList):
        info = info.elements
    if not isinstance(privileges, list):
        if not isinstance(privileges, str):
            raise Exception("The privileges parameter to create_user() must be a list or a string.")
        privileges = [privileges]
    if info is None:
        info = {}
    if isinstance(info, DADict):
        info = info.elements
    if not isinstance(info, dict):
        raise Exception("The info parameter to create_user() must be a dictionary.")
    user, user_email = app.user_manager.find_user_by_email(email)
    if user:
        raise Exception("That e-mail address is already being used.")
    user_auth = UserAuthModel(password=app.user_manager.hash_password(password))
    while True:
        new_social = 'local$' + random_alphanumeric(32)
        existing_user = db.session.execute(select(UserModel).filter_by(social_id=new_social)).first()
        if existing_user:
            continue
        break
    the_user = UserModel(
        active=True,
        nickname=re.sub(r'@.*', '', email),
        social_id=new_social,
        email=email,
        user_auth=user_auth,
        first_name=info.get('first_name', ''),
        last_name=info.get('last_name', ''),
        country=info.get('country', ''),
        subdivisionfirst=info.get('subdivisionfirst', ''),
        subdivisionsecond=info.get('subdivisionsecond', ''),
        subdivisionthird=info.get('subdivisionthird', ''),
        organization=info.get('organization', ''),
        timezone=info.get('timezone', ''),
        language=info.get('language', ''),
        confirmed_at = datetime.datetime.now()
    )
    num_roles = 0
    for role in db.session.execute(select(Role).where(Role.name != 'cron').order_by(Role.id)).scalars():
        if role.name in privileges:
            the_user.roles.append(role)
        num_roles +=1
    if num_roles == 0:
        user_role = db.session.execute(select(Role).filter_by(name='user')).scalar_one()
        the_user.roles.append(user_role)
    db.session.add(user_auth)
    db.session.add(the_user)
    db.session.commit()
    return the_user.id

def set_user_info(**kwargs):
    if current_user.is_anonymous:
        raise Exception("You cannot call set_user_info() unless you are logged in")
    user_id = kwargs.get('user_id', None)
    email = kwargs.get('email', None)
    if user_id is None and email is None:
        user_id = current_user.id
    if not (current_user.has_role('admin')):
        if (user_id is not None and current_user.id != user_id) or (email is not None and current_user.email != email):
            raise Exception("You cannot call set_user_info() unless you are an administrator")
    if user_id is not None:
        user = db.session.execute(select(UserModel).options(db.joinedload(UserModel.roles)).filter_by(id=user_id)).scalar()
    else:
        user = db.session.execute(select(UserModel).options(db.joinedload(UserModel.roles)).filter_by(email=email)).scalar()
    if user is None or user.social_id.startswith('disabled$'):
        raise Exception("User not found")
    for key, val in kwargs.items():
        if key in ('first_name', 'last_name', 'country', 'subdivisionfirst', 'subdivisionsecond', 'subdivisionthird', 'organization', 'timezone', 'language'):
            setattr(user, key, val)
    if 'password' in kwargs:
        if not (current_user.has_role('admin')):
            raise Exception("You cannot call set_user_info() with a password unless you are an administrator.")
        user.user_auth.password = app.user_manager.hash_password(kwargs['password'])
    if 'active' in kwargs:
        if not isinstance(kwargs['active'], bool):
            raise Exception("The active parameter must be True or False")
        if user.id == current_user.id:
            raise Exception("Cannot change active status of the current user.")
        user.active = kwargs['active']
    db.session.commit()
    if 'privileges' in kwargs and isinstance(kwargs['privileges'], (list, tuple, set)):
        if len(kwargs['privileges']) == 0:
            raise Exception("Cannot remove all of a user's privileges.")
        roles_to_add = []
        roles_to_delete = []
        role_names = [role.name for role in user.roles]
        for role in role_names:
            if role not in kwargs['privileges']:
                roles_to_delete.append(role)
        for role in kwargs['privileges']:
            if role not in role_names:
                roles_to_add.append(role)
        for role in roles_to_delete:
            remove_user_privilege(user.id, role)
        for role in roles_to_add:
            add_user_privilege(user.id, role)

@app.route('/api/secret', methods=['GET'])
@cross_origin(origins='*', methods=['GET', 'HEAD'], automatic_options=True)
def api_get_secret():
    if not api_verify(request):
        return jsonify_with_status("Access denied.", 403)
    username = request.args.get('username', None)
    password = request.args.get('password', None)
    if username is None or password is None:
        return jsonify_with_status("A username and password must be supplied", 400)
    try:
        secret = get_secret(str(username), str(password))
    except Exception as err:
        return jsonify_with_status(str(err), 403)
    return jsonify(secret)

def get_secret(username, password, case_sensitive=False):
    if case_sensitive:
        user = db.session.execute(select(UserModel).filter_by(email=username)).scalar()
    else:
        username = re.sub(r'\%', '', username)
        user = db.session.execute(select(UserModel).where(UserModel.email.ilike(username))).scalar()
    if user is None:
        raise Exception("Username not known")
    if app.config['USE_MFA'] and user.otp_secret is not None:
        raise Exception("Secret will not be supplied because two factor authentication is enabled")
    user_manager = current_app.user_manager
    if not user_manager.get_password(user):
        raise Exception("Password not set")
    if not user_manager.verify_password(password, user):
        raise Exception("Incorrect password")
    return pad_to_16(MD5Hash(data=password).hexdigest())

@app.route('/api/users/interviews', methods=['GET', 'DELETE'])
@csrf.exempt
@cross_origin(origins='*', methods=['GET', 'DELETE', 'HEAD'], automatic_options=True)
def api_users_interviews():
    if not api_verify(request, roles=['admin', 'advocate']):
        return jsonify_with_status("Access denied.", 403)
    user_id = request.args.get('user_id', None)
    filename = request.args.get('i', None)
    secret = request.args.get('secret', None)
    tag = request.args.get('tag', None)
    next_id_code = request.args.get('next_id', None)
    if next_id_code:
        try:
            start_id = int(from_safeid(next_id_code))
            assert start_id >= 0
        except:
            start_id = None
    else:
        start_id = None
    if secret is not None:
        secret = str(secret)
    if request.method == 'GET':
        include_dict = true_or_false(request.args.get('include_dictionary', False))
        try:
            (the_list, start_id) = user_interviews(user_id=user_id, secret=secret, exclude_invalid=False, tag=tag, filename=filename, include_dict=include_dict, start_id=start_id)
        except Exception as err:
            return jsonify_with_status("Error getting interview list.  " + str(err), 400)
        if start_id is None:
            next_id = None
        else:
            next_id = safeid(str(start_id))
        return jsonify(dict(next_id=next_id, items=docassemble.base.functions.safe_json(the_list)))
    if request.method == 'DELETE':
        start_id = None
        while True:
            try:
                (the_list, start_id) = user_interviews(user_id=user_id, exclude_invalid=False, tag=tag, filename=filename, include_dict=False, start_id=start_id)
            except:
                return jsonify_with_status("Error reading interview list.", 400)
            for info in the_list:
                user_interviews(user_id=info['user_id'], action='delete', filename=info['filename'], session=info['session'])
            if start_id is None:
                break
        return ('', 204)
    return ('', 204)

@app.route('/api/user/<int:user_id>/interviews', methods=['GET', 'DELETE'])
@csrf.exempt
@cross_origin(origins='*', methods=['GET', 'DELETE', 'HEAD'], automatic_options=True)
def api_user_user_id_interviews(user_id):
    if not api_verify(request):
        return jsonify_with_status("Access denied.", 403)
    if not (current_user.id == user_id or current_user.has_role('admin', 'advocate')):
        return jsonify_with_status("Access denied.", 403)
    filename = request.args.get('i', None)
    secret = request.args.get('secret', None)
    tag = request.args.get('tag', None)
    next_id_code = request.args.get('next_id', None)
    if next_id_code:
        try:
            start_id = int(from_safeid(next_id_code))
            assert start_id >= 0
        except:
            start_id = None
    else:
        start_id = None
    if secret is not None:
        secret = str(secret)
    include_dict = true_or_false(request.args.get('include_dictionary', False))
    if request.method == 'GET':
        try:
            (the_list, start_id) = user_interviews(user_id=user_id, secret=secret, exclude_invalid=False, tag=tag, filename=filename, include_dict=include_dict, start_id=start_id)
        except:
            return jsonify_with_status("Error reading interview list.", 400)
        if start_id is None:
            next_id = None
        else:
            next_id = safeid(str(start_id))
        return jsonify(dict(next_id=next_id, items=docassemble.base.functions.safe_json(the_list)))
    if request.method == 'DELETE':
        start_id = None
        while True:
            try:
                (the_list, start_id) = user_interviews(user_id=user_id, exclude_invalid=False, tag=tag, filename=filename, include_dict=False, start_id=start_id)
            except:
                return jsonify_with_status("Error reading interview list.", 400)
            for info in the_list:
                user_interviews(user_id=info['user_id'], action='delete', filename=info['filename'], session=info['session'])
            if start_id is None:
                break
        return ('', 204)
    return ('', 204)

@app.route('/api/session/back', methods=['POST'])
@csrf.exempt
@cross_origin(origins='*', methods=['POST', 'HEAD'], automatic_options=True)
def api_session_back():
    if not api_verify(request):
        return jsonify_with_status("Access denied.", 403)
    post_data = request.get_json(silent=True)
    if post_data is None:
        post_data = request.form.copy()
    yaml_filename = post_data.get('i', None)
    session_id = post_data.get('session', None)
    secret = str(post_data.get('secret', None))
    reply_with_question = true_or_false(post_data.get('question', True))
    if yaml_filename is None or session_id is None:
        return jsonify_with_status("Parameters i and session are required.", 400)
    try:
        data = go_back_in_session(yaml_filename, session_id, secret=secret, return_question=reply_with_question)
    except Exception as the_err:
        return jsonify_with_status(str(the_err), 400)
    if data is None:
        return ('', 204)
    if data.get('questionType', None) is 'response':
        return data['response']
    return jsonify(**data)

def transform_json_variables(obj):
    if isinstance(obj, str):
        if re.search(r'^[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]T', obj):
            try:
                return docassemble.base.util.as_datetime(dateutil.parser.parse(obj))
            except:
                pass
        else:
            return obj
    if isinstance(obj, (bool, int, float)):
        return obj
    if isinstance(obj, dict):
        if '_class' in obj and obj['_class'] == 'type' and 'name' in obj and isinstance(obj['name'], str) and valid_python_exp.match(obj['name']):
            if '.' in obj['name']:
                the_module = re.sub(r'\.[^\.]+$', '', obj['name'])
            else:
                the_module = None
            try:
                if the_module:
                    importlib.import_module(the_module)
                return eval(obj['name'])
            except Exception as err:
                logmessage("transform_json_variables: " + err.__class__.__name__ + ": " + str(err))
                return None
        if '_class' in obj and isinstance(obj['_class'], str) and 'instanceName' in obj and valid_python_exp.match(obj['_class']) and isinstance(obj['instanceName'], str):
            the_module = re.sub(r'\.[^\.]+$', '', obj['_class'])
            try:
                importlib.import_module(the_module)
                the_class = eval(obj['_class'])
                new_obj = the_class(obj['instanceName'])
                for key, val in obj.items():
                    if key == '_class':
                        continue
                    setattr(new_obj, key, transform_json_variables(val))
                return new_obj
            except Exception as err:
                logmessage("transform_json_variables: " + err.__class__.__name__ + ": " + str(err))
                return None
        else:
            new_dict = {}
            for key, val in obj.items():
                new_dict[transform_json_variables(key)] = transform_json_variables(val)
            return new_dict
    if isinstance(obj, list):
        return [transform_json_variables(val) for val in obj]
    if isinstance(obj, set):
        return set(transform_json_variables(val) for val in obj)
    return obj

@app.route('/api/session', methods=['GET', 'POST', 'DELETE'])
@csrf.exempt
@cross_origin(origins='*', methods=['GET', 'POST', 'DELETE', 'HEAD'], automatic_options=True)
def api_session():
    if not api_verify(request):
        return jsonify_with_status("Access denied.", 403)
    if request.method == 'GET':
        yaml_filename = request.args.get('i', None)
        session_id = request.args.get('session', None)
        secret = request.args.get('secret', None)
        if secret is not None:
            secret = str(secret)
        if yaml_filename is None or session_id is None:
            return jsonify_with_status("Parameters i and session are required.", 400)
        try:
            variables = get_session_variables(yaml_filename, session_id, secret=secret)
        except Exception as the_err:
            return jsonify_with_status(str(the_err), 400)
        return jsonify(variables)
    if request.method == 'POST':
        post_data = request.get_json(silent=True)
        if post_data is None:
            post_data = request.form.copy()
        yaml_filename = post_data.get('i', None)
        session_id = post_data.get('session', None)
        secret = str(post_data.get('secret', None))
        question_name = post_data.get('question_name', None)
        treat_as_raw = true_or_false(post_data.get('raw', False))
        advance_progress_meter = true_or_false(post_data.get('advance_progress_meter', False))
        post_setting = not true_or_false(post_data.get('overwrite', False))
        reply_with_question = true_or_false(post_data.get('question', True))
        if yaml_filename is None or session_id is None:
            return jsonify_with_status("Parameters i and session are required.", 400)
        if 'variables' in post_data and isinstance(post_data['variables'], dict):
            variables = post_data['variables']
        else:
            try:
                variables = json.loads(post_data.get('variables', '{}'))
            except:
                return jsonify_with_status("Malformed variables.", 400)
        if not treat_as_raw:
            variables = transform_json_variables(variables)
        if 'file_variables' in post_data and isinstance(post_data['file_variables'], dict):
            file_variables = post_data['file_variables']
        else:
            try:
                file_variables = json.loads(post_data.get('file_variables', '{}'))
            except:
                return jsonify_with_status("Malformed list of file variables.", 400)
        if 'delete_variables' in post_data and isinstance(post_data['delete_variables'], list):
            del_variables = post_data['delete_variables']
        else:
            try:
                del_variables = json.loads(post_data.get('delete_variables', '[]'))
            except:
                return jsonify_with_status("Malformed list of delete variables.", 400)
        if 'event_list' in post_data and isinstance(post_data['event_list'], list):
            event_list = post_data['event_list']
        else:
            try:
                event_list = json.loads(post_data.get('event_list', '[]'))
                assert isinstance(event_list, list)
            except:
                return jsonify_with_status("Malformed event list.", 400)
        if not isinstance(variables, dict):
            return jsonify_with_status("Variables data is not a dict.", 400)
        if not isinstance(file_variables, dict):
            return jsonify_with_status("File variables data is not a dict.", 400)
        if not isinstance(del_variables, list):
            return jsonify_with_status("Delete variables data is not a list.", 400)
        if not isinstance(event_list, list):
            return jsonify_with_status("Event list data is not a list.", 400)
        files = []
        literal_variables = {}
        for filekey in request.files:
            if filekey not in file_variables:
                file_variables[filekey] = filekey
            the_files = request.files.getlist(filekey)
            files_to_process = []
            if the_files:
                for the_file in the_files:
                    filename = secure_filename(the_file.filename)
                    file_number = get_new_file_number(session_id, filename, yaml_file_name=yaml_filename)
                    extension, mimetype = get_ext_and_mimetype(filename)
                    saved_file = SavedFile(file_number, extension=extension, fix=True, should_not_exist=True)
                    temp_file = tempfile.NamedTemporaryFile(prefix="datemp", suffix='.' + extension, delete=False)
                    the_file.save(temp_file.name)
                    process_file(saved_file, temp_file.name, mimetype, extension)
                    files_to_process.append((filename, file_number, mimetype, extension))
            file_field = file_variables[filekey]
            if illegal_variable_name(file_field):
                return jsonify_with_status("Malformed file variable.", 400)
            if len(files_to_process) > 0:
                elements = []
                indexno = 0
                for (filename, file_number, mimetype, extension) in files_to_process:
                    elements.append("docassemble.base.util.DAFile(" + repr(file_field + '[' + str(indexno) + ']') + ", filename=" + repr(filename) + ", number=" + str(file_number) + ", make_pngs=True, mimetype=" + repr(mimetype) + ", extension=" + repr(extension) + ")")
                    indexno += 1
                literal_variables[file_field] = "docassemble.base.util.DAFileList(" + repr(file_field) + ", elements=[" + ", ".join(elements) + "])"
            else:
                literal_variables[file_field] = "None"
        try:
            data = set_session_variables(yaml_filename, session_id, variables, secret=secret, return_question=reply_with_question, literal_variables=literal_variables, del_variables=del_variables, question_name=question_name, event_list=event_list, advance_progress_meter=advance_progress_meter, post_setting=post_setting)
        except Exception as the_err:
            return jsonify_with_status(str(the_err), 400)
        if data is None:
            return ('', 204)
        if data.get('questionType', None) is 'response':
            return data['response']
        return jsonify(**data)
    if request.method == 'DELETE':
        yaml_filename = request.args.get('i', None)
        session_id = request.args.get('session', None)
        if yaml_filename is None or session_id is None:
            return jsonify_with_status("Parameters i and session are required.", 400)
        user_interviews(action='delete', filename=yaml_filename, session=session_id)
        return ('', 204)
    return ('', 204)

@app.route('/api/file/<int:file_number>', methods=['GET'])
@cross_origin(origins='*', methods=['GET', 'HEAD'], automatic_options=True)
def api_file(file_number):
    if not api_verify(request):
        return jsonify_with_status("Access denied.", 403)
    if request.method == 'GET':
        yaml_filename = request.args.get('i', None)
        session_id = request.args.get('session', None)
        number = re.sub(r'[^0-9]', '', str(file_number))
        privileged = bool(current_user.is_authenticated and current_user.has_role('admin', 'advocate'))
        try:
            file_info = get_info_from_file_number(number, privileged=privileged, uids=get_session_uids())
        except:
            return ('File not found', 404)
        if 'path' not in file_info:
            return ('File not found', 404)
        else:
            if 'extension' in request.args:
                extension = werkzeug.utils.secure_filename(request.args['extension'])
                if os.path.isfile(file_info['path'] + '.' + extension):
                    the_path = file_info['path'] + '.' + extension
                    extension, mimetype = get_ext_and_mimetype(file_info['path'] + '.' + extension)
                else:
                    return ('File not found', 404)
            elif 'filename' in request.args:
                the_filename = secure_filename_spaces_ok(request.args['filename'])
                if os.path.isfile(os.path.join(os.path.dirname(file_info['path']), the_filename)):
                    the_path = os.path.join(os.path.dirname(file_info['path']), the_filename)
                    extension, mimetype = get_ext_and_mimetype(the_filename)
                else:
                    return ('File not found', 404)
            else:
                the_path = file_info['path']
                mimetype = file_info['mimetype']
            if not os.path.isfile(the_path):
                return ('File not found', 404)
            response = send_file(the_path, mimetype=mimetype)
            response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
            return response
        return ('File not found', 404)

def get_session_variables(yaml_filename, session_id, secret=None, simplify=True, use_lock=False):
    if use_lock:
        obtain_lock(session_id, yaml_filename)
    #sys.stderr.write("get_session_variables: fetch_user_dict\n")
    if secret is None:
        secret = docassemble.base.functions.this_thread.current_info.get('secret', None)
    try:
        steps, user_dict, is_encrypted = fetch_user_dict(session_id, yaml_filename, secret=str(secret))
    except Exception as the_err:
        if use_lock:
            release_lock(session_id, yaml_filename)
        raise Exception("Unable to decrypt interview dictionary: " + str(the_err))
    if use_lock:
        release_lock(session_id, yaml_filename)
    if user_dict is None:
        raise Exception("Unable to obtain interview dictionary.")
    if simplify:
        variables = docassemble.base.functions.serializable_dict(user_dict, include_internal=True)
        #variables['_internal'] = docassemble.base.functions.serializable_dict(user_dict['_internal'])
        return variables
    return user_dict

def go_back_in_session(yaml_filename, session_id, secret=None, return_question=False, use_lock=False, encode=False):
    if use_lock:
        obtain_lock(session_id, yaml_filename)
    try:
        steps, user_dict, is_encrypted = fetch_user_dict(session_id, yaml_filename, secret=secret)
    except:
        if use_lock:
            release_lock(session_id, yaml_filename)
        raise Exception("Unable to decrypt interview dictionary.")
    if user_dict is None:
        if use_lock:
            release_lock(session_id, yaml_filename)
        raise Exception("Unable to obtain interview dictionary.")
    if steps == 1:
        if use_lock:
            release_lock(session_id, yaml_filename)
        raise Exception("Cannot go back.")
    old_user_dict = user_dict
    steps, user_dict, is_encrypted = fetch_previous_user_dict(session_id, yaml_filename, secret)
    if user_dict is None:
        if use_lock:
            release_lock(session_id, yaml_filename)
        raise Exception("Unable to obtain interview dictionary.")
    if return_question:
        try:
            data = get_question_data(yaml_filename, session_id, secret, use_lock=False, user_dict=user_dict, steps=steps, is_encrypted=is_encrypted, old_user_dict=old_user_dict, encode=encode)
        except Exception as the_err:
            if use_lock:
                release_lock(session_id, yaml_filename)
            raise Exception("Problem getting current question:" + str(the_err))
    else:
        data = None
    if use_lock:
        release_lock(session_id, yaml_filename)
    return data

def set_session_variables(yaml_filename, session_id, variables, secret=None, return_question=False, literal_variables=None, del_variables=None, question_name=None, event_list=None, advance_progress_meter=False, post_setting=True, use_lock=False, encode=False):
    if use_lock:
        obtain_lock(session_id, yaml_filename)
    device_id = docassemble.base.functions.this_thread.current_info['user']['device_id']
    session_uid = docassemble.base.functions.this_thread.current_info['user']['session_uid']
    if secret is None:
        secret = docassemble.base.functions.this_thread.current_info.get('secret', None)
    try:
        steps, user_dict, is_encrypted = fetch_user_dict(session_id, yaml_filename, secret=secret)
    except:
        if use_lock:
            release_lock(session_id, yaml_filename)
        raise Exception("Unable to decrypt interview dictionary.")
    vars_set = set()
    old_values = {}
    if user_dict is None:
        if use_lock:
            release_lock(session_id, yaml_filename)
        raise Exception("Unable to obtain interview dictionary.")
    pre_assembly_necessary = False
    for key, val in variables.items():
        if contains_volatile.search(key):
            pre_assembly_necessary = True
            break
    if pre_assembly_necessary is False and literal_variables is not None:
        for key, val in literal_variables.items():
            if contains_volatile.search(key):
                pre_assembly_necessary = True
                break
    if pre_assembly_necessary is False and del_variables is not None:
        for key in del_variables:
            if contains_volatile.search(key):
                pre_assembly_necessary = True
                break
    if pre_assembly_necessary:
        interview = docassemble.base.interview_cache.get_interview(yaml_filename)
        ci = current_info(yaml=yaml_filename, req=request, secret=secret, device_id=device_id, session_uid=session_uid)
        ci['session'] = session_id
        ci['encrypted'] = is_encrypted
        ci['secret'] = secret
        interview_status = docassemble.base.parse.InterviewStatus(current_info=ci)
        interview.assemble(user_dict, interview_status)
    try:
        for key, val in variables.items():
            if illegal_variable_name(key):
                raise Exception("Illegal value as variable name.")
            if isinstance(val, (str, bool, int, float, NoneType)):
                exec(str(key) + ' = ' + repr(val), user_dict)
            else:
                if key == '_xxxtempvarxxx':
                    continue
                user_dict['_xxxtempvarxxx'] = copy.deepcopy(val)
                exec(str(key) + ' = _xxxtempvarxxx', user_dict)
                del user_dict['_xxxtempvarxxx']
            process_set_variable(str(key), user_dict, vars_set, old_values)
    except Exception as the_err:
        if '_xxxtempvarxxx' in user_dict:
            del user_dict['_xxxtempvarxxx']
        if use_lock:
            release_lock(session_id, yaml_filename)
        raise Exception("Problem setting variables:" + str(the_err))
    if literal_variables is not None:
        exec('import docassemble.base.util', user_dict)
        for key, val in literal_variables.items():
            if illegal_variable_name(key):
                if use_lock:
                    release_lock(session_id, yaml_filename)
                raise Exception("Illegal value as variable name.")
            exec(str(key) + ' = ' + val, user_dict)
            process_set_variable(str(key), user_dict, vars_set, old_values)
    if question_name is not None:
        interview = docassemble.base.interview_cache.get_interview(yaml_filename)
        if question_name in interview.questions_by_name:
            interview.questions_by_name[question_name].mark_as_answered(user_dict)
        else:
            raise Exception("Problem marking question as completed")
    if del_variables is not None:
        try:
            for key in del_variables:
                if illegal_variable_name(key):
                    if use_lock:
                        release_lock(session_id, yaml_filename)
                    raise Exception("Illegal value as variable name.")
                exec('del ' + str(key), user_dict)
        except Exception as the_err:
            if use_lock:
                release_lock(session_id, yaml_filename)
            raise Exception("Problem deleting variables: " + str(the_err))
    session_uid = docassemble.base.functions.this_thread.current_info['user']['session_uid']
    #if 'event_stack' in user_dict['_internal']:
    #    logmessage("Event stack starting as: " + repr(user_dict['_internal']['event_stack']))
    #else:
    #    logmessage("No event stack.")
    if event_list is not None and len(event_list) and 'event_stack' in user_dict['_internal'] and session_uid in user_dict['_internal']['event_stack'] and len(user_dict['_internal']['event_stack'][session_uid]):
        for event_name in event_list:
            if user_dict['_internal']['event_stack'][session_uid][0]['action'] == event_name:
                user_dict['_internal']['event_stack'][session_uid].pop(0)
                #logmessage("Popped " + str(event_name))
            if len(user_dict['_internal']['event_stack'][session_uid]) == 0:
                break
    if len(vars_set) > 0 and 'event_stack' in user_dict['_internal'] and session_uid in user_dict['_internal']['event_stack'] and len(user_dict['_internal']['event_stack'][session_uid]):
        for var_name in vars_set:
            if user_dict['_internal']['event_stack'][session_uid][0]['action'] == var_name:
                user_dict['_internal']['event_stack'][session_uid].pop(0)
                #logmessage("Popped " + str(var_name))
            if len(user_dict['_internal']['event_stack'][session_uid]) == 0:
                break
    if question_name is not None:
        for var_name in vars_set:
            if var_name in interview.invalidation_todo or var_name in interview.onchange_todo:
                interview.invalidate_dependencies(var_name, user_dict, old_values)
            try:
                del user_dict['_internal']['dirty'][var_name]
            except:
                pass
    #if 'event_stack' in user_dict['_internal']:
    #    logmessage("Event stack now: " + repr(user_dict['_internal']['event_stack']))
    if post_setting:
        steps += 1
    if return_question:
        try:
            data = get_question_data(yaml_filename, session_id, secret, use_lock=False, user_dict=user_dict, steps=steps, is_encrypted=is_encrypted, post_setting=post_setting, advance_progress_meter=advance_progress_meter, encode=encode)
        except Exception as the_err:
            if use_lock:
                release_lock(session_id, yaml_filename)
            raise Exception("Problem getting current question:" + str(the_err))
    else:
        data = None
    if not return_question:
        save_user_dict(session_id, user_dict, yaml_filename, secret=secret, encrypt=is_encrypted, changed=post_setting, steps=steps)
        if 'multi_user' in vars_set:
            if user_dict.get('multi_user', False) is True and is_encrypted is True:
                decrypt_session(secret, user_code=session_id, filename=yaml_filename)
                is_encrypted = False
            if user_dict.get('multi_user', False) is False and is_encrypted is False:
                encrypt_session(secret, user_code=session_id, filename=yaml_filename)
                is_encrypted = True
    if use_lock:
        release_lock(session_id, yaml_filename)
    return data

@app.route('/api/session/new', methods=['GET'])
@cross_origin(origins='*', methods=['GET', 'HEAD'], automatic_options=True)
def api_session_new():
    if not api_verify(request):
        return jsonify_with_status("Access denied.", 403)
    yaml_filename = request.args.get('i', None)
    if yaml_filename is None:
        return jsonify_with_status("Parameter i is required.", 400)
    secret = request.args.get('secret', None)
    if secret is None:
        new_secret = True
        secret = random_string(16)
    else:
        new_secret = False
    secret = str(secret)
    url_args = {}
    for argname in request.args:
        if argname in ('i', 'secret', 'key'):
            continue
        if re.match('[A-Za-z_][A-Za-z0-9_]*', argname):
            url_args[argname] = request.args[argname]
    try:
        (encrypted, session_id) = create_new_interview(yaml_filename, secret, url_args=url_args, req=request)
    except Exception as err:
        return jsonify_with_status(err.__class__.__name__ + ': ' + str(err), 400)
    if encrypted and new_secret:
        return jsonify(dict(session=session_id, i=yaml_filename, secret=secret, encrypted=encrypted))
    else:
        return jsonify(dict(session=session_id, i=yaml_filename, encrypted=encrypted))

def create_new_interview(yaml_filename, secret, url_args=None, referer=None, req=None):
    interview = docassemble.base.interview_cache.get_interview(yaml_filename)
    if current_user.is_anonymous:
        if not interview.allowed_to_initiate(is_anonymous=True):
            raise Exception('Insufficient permissions to run this interview.')
        if not interview.allowed_to_access(is_anonymous=True):
            raise Exception('Insufficient permissions to run this interview.')
    else:
        if (not current_user.has_role('admin')) and (not interview.allowed_to_initiate(has_roles=[role.name for role in current_user.roles])):
            raise Exception('Insufficient permissions to run this interview.')
        if not interview.allowed_to_access(has_roles=[role.name for role in current_user.roles]):
            raise Exception('Insufficient permissions to run this interview.')
    if req is None:
        req = request
    if secret is None:
        secret = random_string(16)
    session_id, user_dict = reset_session(yaml_filename, secret)
    add_referer(user_dict, referer=referer)
    if url_args:
        for key, val in url_args.items():
            if isinstance(val, str):
                val = val.encode('unicode_escape').decode()
            exec("url_args['" + key + "'] = " + repr(val), user_dict)
    device_id = docassemble.base.functions.this_thread.current_info['user']['device_id']
    session_uid = docassemble.base.functions.this_thread.current_info['user']['session_uid']
    ci = current_info(yaml=yaml_filename, req=req, secret=secret, device_id=device_id, session_uid=session_uid)
    ci['session'] = session_id
    ci['encrypted'] = True
    ci['secret'] = secret
    interview_status = docassemble.base.parse.InterviewStatus(current_info=ci)
    interview_status.checkin = True
    tbackup = docassemble.base.functions.backup_thread_variables()
    sbackup = backup_session()
    try:
        interview.assemble(user_dict, interview_status)
    except DAErrorMissingVariable as err:
        pass
    except Exception as e:
        release_lock(session_id, yaml_filename)
        restore_session(sbackup)
        docassemble.base.functions.restore_thread_variables(tbackup)
        if hasattr(e, 'traceback'):
            the_trace = e.traceback
        else:
            the_trace = traceback.format_exc()
        raise Exception("create_new_interview: failure to assemble interview: " + e.__class__.__name__ + ": " + str(e) + "\n" + str(the_trace))
    restore_session(sbackup)
    docassemble.base.functions.restore_thread_variables(tbackup)
    encrypted = not bool(user_dict.get('multi_user', False) is True)
    save_user_dict(session_id, user_dict, yaml_filename, secret=secret, encrypt=encrypted, changed=False, steps=1)
    save_user_dict_key(session_id, yaml_filename)
    release_lock(session_id, yaml_filename)
    return (encrypted, session_id)

@app.route('/api/session/question', methods=['GET'])
@cross_origin(origins='*', methods=['GET', 'HEAD'], automatic_options=True)
def api_session_question():
    if not api_verify(request):
        return jsonify_with_status("Access denied.", 403)
    yaml_filename = request.args.get('i', None)
    session_id = request.args.get('session', None)
    secret = request.args.get('secret', None)
    if secret is not None:
        secret = str(secret)
    if yaml_filename is None or session_id is None:
        return jsonify_with_status("Parameters i and session are required.", 400)
    try:
        data = get_question_data(yaml_filename, session_id, secret)
    except Exception as err:
        return jsonify_with_status(str(err), 400)
    if data.get('questionType', None) == 'response':
        return data['response']
    return jsonify(**data)

def get_question_data(yaml_filename, session_id, secret, use_lock=True, user_dict=None, steps=None, is_encrypted=None, old_user_dict=None, save=True, post_setting=False, advance_progress_meter=False, action=None, encode=False):
    if use_lock:
        obtain_lock(session_id, yaml_filename)
    if user_dict is None:
        try:
            steps, user_dict, is_encrypted = fetch_user_dict(session_id, yaml_filename, secret=secret)
        except Exception as err:
            if use_lock:
                release_lock(session_id, yaml_filename)
            raise Exception("Unable to obtain interview dictionary: " + str(err))
    interview = docassemble.base.interview_cache.get_interview(yaml_filename)
    device_id = docassemble.base.functions.this_thread.current_info['user']['device_id']
    session_uid = docassemble.base.functions.this_thread.current_info['user']['session_uid']
    ci = current_info(yaml=yaml_filename, req=request, secret=secret, device_id=device_id, action=action, session_uid=session_uid)
    ci['session'] = session_id
    ci['encrypted'] = is_encrypted
    ci['secret'] = secret
    interview_status = docassemble.base.parse.InterviewStatus(current_info=ci)
    #interview_status.checkin = True
    tbackup = docassemble.base.functions.backup_thread_variables()
    sbackup = backup_session()
    try:
        interview.assemble(user_dict, interview_status=interview_status, old_user_dict=old_user_dict)
    except DAErrorMissingVariable as err:
        if use_lock:
            #save_user_dict(session_id, user_dict, yaml_filename, secret=secret, encrypt=is_encrypted, changed=False, steps=steps)
            release_lock(session_id, yaml_filename)
        restore_session(sbackup)
        docassemble.base.functions.restore_thread_variables(tbackup)
        return dict(questionType='undefined_variable', variable=err.variable, message_log=docassemble.base.functions.get_message_log())
    except Exception as e:
        if use_lock:
            release_lock(session_id, yaml_filename)
        restore_session(sbackup)
        docassemble.base.functions.restore_thread_variables(tbackup)
        raise Exception("get_question_data: failure to assemble interview: " + e.__class__.__name__ + ": " + str(e))
    save_status = docassemble.base.functions.this_thread.misc.get('save_status', 'new')
    restore_session(sbackup)
    docassemble.base.functions.restore_thread_variables(tbackup)
    try:
        the_section = user_dict['nav'].get_section()
        the_section_display = user_dict['nav'].get_section(display=True)
        the_sections = user_dict['nav'].get_sections()
    except:
        the_section = None
        the_section_display = None
        the_sections = []
    if advance_progress_meter:
        if interview.use_progress_bar and interview_status.question.progress is None and save_status == 'new':
            advance_progress(user_dict, interview)
        if interview.use_progress_bar and interview_status.question.progress is not None and (user_dict['_internal']['progress'] is None or interview_status.question.interview.options.get('strict progress', False) or interview_status.question.progress > user_dict['_internal']['progress']):
            user_dict['_internal']['progress'] = interview_status.question.progress
    if save:
        save_user_dict(session_id, user_dict, yaml_filename, secret=secret, encrypt=is_encrypted, changed=post_setting, steps=steps)
        if user_dict.get('multi_user', False) is True and is_encrypted is True:
            decrypt_session(secret, user_code=session_id, filename=yaml_filename)
            is_encrypted = False
        if user_dict.get('multi_user', False) is False and is_encrypted is False:
            encrypt_session(secret, user_code=session_id, filename=yaml_filename)
            is_encrypted = True
    if use_lock:
        release_lock(session_id, yaml_filename)
    if interview_status.question.question_type == "response":
        if hasattr(interview_status.question, 'response_code'):
            resp_code = interview_status.question.response_code
        else:
            resp_code = 200
        if hasattr(interview_status.question, 'all_variables'):
            if hasattr(interview_status.question, 'include_internal'):
                include_internal = interview_status.question.include_internal
            else:
                include_internal = False
            response_to_send = make_response(docassemble.base.functions.dict_as_json(user_dict, include_internal=include_internal).encode('utf-8'), resp_code)
        elif hasattr(interview_status.question, 'binaryresponse'):
            response_to_send = make_response(interview_status.question.binaryresponse, resp_code)
        else:
            response_to_send = make_response(interview_status.questionText.encode('utf-8'), resp_code)
        response_to_send.headers['Content-Type'] = interview_status.extras['content_type']
        return dict(questionType='response', response=response_to_send)
    if interview_status.question.question_type == "sendfile":
        if interview_status.question.response_file is not None:
            the_path = interview_status.question.response_file.path()
        else:
            return jsonify_with_status("Could not send file because the response was None", 404)
        if not os.path.isfile(the_path):
            return jsonify_with_status("Could not send file because " + str(the_path) + " not found", 404)
        response_to_send = send_file(the_path, mimetype=interview_status.extras['content_type'])
        response_to_send.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
        return dict(questionType='response', response=response_to_send)
    if interview_status.question.language != '*':
        interview_language = interview_status.question.language
    else:
        interview_language = DEFAULT_LANGUAGE
    title_info = interview.get_title(user_dict, status=interview_status, converter=lambda content, part: title_converter(content, part, interview_status))
    interview_status.exit_url = title_info.get('exit url', None)
    interview_status.exit_link = title_info.get('exit link', 'exit')
    interview_status.exit_label = title_info.get('exit label', word('Exit'))
    interview_status.title = title_info.get('full', default_title)
    interview_status.display_title = title_info.get('logo', interview_status.title)
    interview_status.tabtitle = title_info.get('tab', interview_status.title)
    interview_status.short_title = title_info.get('short', title_info.get('full', default_short_title))
    interview_status.display_short_title = title_info.get('short logo', title_info.get('logo', interview_status.short_title))
    interview_status.title_url = title_info.get('title url', None)
    interview_status.title_url_opens_in_other_window = title_info.get('title url opens in other window', True)
    interview_status.nav_item = title_info.get('navigation bar html', '')
    the_main_page_parts = main_page_parts.get(interview_language, main_page_parts.get('*'))
    interview_status.pre = title_info.get('pre', the_main_page_parts['main page pre'])
    interview_status.post = title_info.get('post', the_main_page_parts['main page post'])
    interview_status.footer = title_info.get('footer', the_main_page_parts['main page footer'] or get_part('global footer'))
    if interview_status.footer:
        interview_status.footer = re.sub(r'</?p.*?>', '', str(interview_status.footer), flags=re.IGNORECASE).strip()
        if interview_status.footer == 'off':
            interview_status.footer = ''
    interview_status.submit = title_info.get('submit', the_main_page_parts['main page submit'])
    interview_status.back = title_info.get('back button label', the_main_page_parts['main page back button label'] or interview_status.question.back())
    interview_status.cornerback = title_info.get('corner back button label', the_main_page_parts['main page corner back button label'] or interview_status.question.cornerback())
    bootstrap_theme = interview_status.question.interview.get_bootstrap_theme()
    if steps is None:
        steps = user_dict['_internal']['steps']
    allow_going_back = bool(interview_status.question.can_go_back and (steps is None or (steps - user_dict['_internal']['steps_offset']) > 1))
    data = dict(browser_title=interview_status.tabtitle, exit_link=interview_status.exit_link, exit_url=interview_status.exit_url, exit_label=interview_status.exit_label, title=interview_status.title, display_title=interview_status.display_title, short_title=interview_status.short_title, lang=interview_language, steps=steps, allow_going_back=allow_going_back, message_log=docassemble.base.functions.get_message_log(), section=the_section, display_section=the_section_display, sections=the_sections)
    if allow_going_back:
        data['cornerBackButton'] = interview_status.cornerback
    data.update(interview_status.as_data(user_dict, encode=encode))
    if 'source' in data:
        data['source']['varsLink'] = url_for('get_variables', i=yaml_filename)
        data['source']['varsLabel'] = word('Show variables and values')
    #if interview_status.question.question_type == "review" and len(interview_status.question.fields_used):
    #    next_action_review = dict(action=list(interview_status.question.fields_used)[0], arguments={})
    #else:
    #    next_action_review = None
    if 'reload_after' in interview_status.extras:
        reload_after = 1000 * int(interview_status.extras['reload_after'])
    else:
        reload_after = 0
    #if next_action_review:
    #    data['next_action'] = next_action_review
    data['interview_options'] = interview_status.question.interview.options
    if reload_after and reload_after > 0:
        data['reload_after'] = reload_after
    for key in list(data.keys()):
        if key == "_question_name":
            data['questionName'] = data[key]
            del data[key]
        elif key.startswith('_'):
            del data[key]
    data['menu'] = {'items': []}
    menu_items = data['menu']['items']
    if 'menu_items' in interview_status.extras:
        if not isinstance(interview_status.extras['menu_items'], list):
            menu_items.append({'anchor': word("Error: menu_items is not a Python list")})
        elif len(interview_status.extras['menu_items']) > 0:
            for menu_item in interview_status.extras['menu_items']:
                if not (isinstance(menu_item, dict) and 'url' in menu_item and 'label' in menu_item):
                    menu_items.append({'anchor': word("Error: menu item is not a Python dict with keys of url and label")})
                else:
                    match_action = re.search(r'^\?action=([^\&]+)', menu_item['url'])
                    if match_action:
                        menu_items.append({'href': menu_item['url'], 'action': match_action.group(1), 'anchor': menu_item['label']})
                    else:
                        menu_items.append({'href': menu_item['url'], 'anchor': menu_item['label']})
    if ALLOW_REGISTRATION:
        sign_in_text = word('Sign in or sign up to save answers')
    else:
        sign_in_text = word('Sign in to save answers')
    if daconfig.get('resume interview after login', False):
        login_url = url_for('user.login', next=url_for('index', i=yaml_filename))
    else:
        login_url = url_for('user.login')
    if interview_status.question.interview.consolidated_metadata.get('show login', SHOW_LOGIN):
        if current_user.is_anonymous:
            if len(menu_items) > 0:
                data['menu']['top'] = {'anchor': word("Menu")}
                menu_items.append({'href': login_url, 'anchor': sign_in_text})
            else:
                data['menu']['top'] = {'href': login_url, 'anchor': sign_in_text}
        else:
            if len(menu_items) == 0 and interview_status.question.interview.options.get('hide standard menu', False):
                data['menu']['top'] = {'anchor': (current_user.email if current_user.email else re.sub(r'.*\$', '', current_user.social_id))}
            else:
                data['menu']['top'] = {'anchor': current_user.email if current_user.email else re.sub(r'.*\$', '', current_user.social_id)}
                if not interview_status.question.interview.options.get('hide standard menu', False):
                    if current_user.has_role('admin', 'developer') and interview_status.question.interview.debug:
                        menu_items.append({'href': '#source', 'title': word("How this question came to be asked"), 'anchor': word('Source')})
                    if current_user.has_role('admin', 'advocate') and app.config['ENABLE_MONITOR']:
                        menu_items.append({'href': url_for('monitor'), 'anchor': word('Monitor')})
                    if current_user.has_role('admin', 'developer', 'trainer'):
                        menu_items.append({'href': url_for('train'), 'anchor': word('Train')})
                    if current_user.has_role('admin', 'developer'):
                        if app.config['ALLOW_UPDATES']:
                            menu_items.append({'href': url_for('update_package'), 'anchor': word('Package Management')})
                        menu_items.append({'href': url_for('logs'), 'anchor': word('Logs')})
                        if app.config['ENABLE_PLAYGROUND']:
                            menu_items.append({'href': url_for('playground_page'), 'anchor': word('Playground')})
                        menu_items.append({'href': url_for('utilities'), 'anchor': word('Utilities')})
                        if current_user.has_role('admin'):
                            menu_items.append({'href': url_for('user_list'), 'anchor': word('User List')})
                            menu_items.append({'href': url_for('config_page'), 'anchor': word('Configuration')})
                    if app.config['SHOW_DISPATCH']:
                        menu_items.append({'href': url_for('interview_start'), 'anchor': word('Available Interviews')})
                    for item in app.config['ADMIN_INTERVIEWS']:
                        if item.can_use() and docassemble.base.functions.this_thread.current_info.get('yaml_filename', '') != item.interview:
                            menu_items.append({'href': url_for('index'), 'anchor': item.get_title(docassemble.base.functions.get_language())})
                    if app.config['SHOW_MY_INTERVIEWS'] or current_user.has_role('admin'):
                        menu_items.append({'href': url_for('interview_list'), 'anchor': word('My Interviews')})
                    if current_user.has_role('admin', 'developer'):
                        menu_items.append({'href': url_for('user_profile_page'), 'anchor': word('Profile')})
                    else:
                        if app.config['SHOW_PROFILE'] or current_user.has_role('admin'):
                            menu_items.append({'href': url_for('user_profile_page'), 'anchor': word('Profile')})
                        else:
                            menu_items.append({'href': url_for('user.change_password'), 'anchor': word('Change Password')})
                    menu_items.append({'href': url_for('user.logout'), 'anchor': word('Sign Out')})
    else:
        if len(menu_items) > 0:
            data['menu']['top'] = {'anchor': word("Menu")}
            if not interview_status.question.interview.options.get('hide standard menu', False):
                menu_items.append({'href': exit_href(interview_status), 'anchor': interview_status.exit_label})
        else:
            data['menu']['top'] = {'href': exit_href(interview_status), 'anchor': interview_status.exit_label}
    #logmessage("Ok returning")
    return data

@app.route('/api/session/action', methods=['POST'])
@csrf.exempt
@cross_origin(origins='*', methods=['POST', 'HEAD'], automatic_options=True)
def api_session_action():
    if not api_verify(request):
        return jsonify_with_status("Access denied.", 403)
    post_data = request.get_json(silent=True)
    if post_data is None:
        post_data = request.form.copy()
    yaml_filename = post_data.get('i', None)
    session_id = post_data.get('session', None)
    secret = post_data.get('secret', None)
    action = post_data.get('action', None)
    persistent = true_or_false(post_data.get('persistent', False))
    if yaml_filename is None or session_id is None or action is None:
        return jsonify_with_status("Parameters i, session, and action are required.", 400)
    secret = str(secret)
    if 'arguments' in post_data:
        if isinstance(post_data['arguments'], dict):
            arguments = post_data['arguments']
        else:
            try:
                arguments = json.loads(post_data['arguments'])
            except:
                return jsonify_with_status("Malformed arguments.", 400)
            if not isinstance(arguments, dict):
                return jsonify_with_status("Arguments data is not a dict.", 400)
    else:
        arguments = {}
    obtain_lock(session_id, yaml_filename)
    try:
        steps, user_dict, is_encrypted = fetch_user_dict(session_id, yaml_filename, secret=secret)
    except:
        release_lock(session_id, yaml_filename)
        return jsonify_with_status("Unable to obtain interview dictionary.", 400)
    interview = docassemble.base.interview_cache.get_interview(yaml_filename)
    device_id = docassemble.base.functions.this_thread.current_info['user']['device_id']
    session_uid = docassemble.base.functions.this_thread.current_info['user']['session_uid']
    ci = current_info(yaml=yaml_filename, req=request, action=dict(action=action, arguments=arguments), secret=secret, device_id=device_id, session_uid=session_uid)
    ci['session'] = session_id
    ci['encrypted'] = is_encrypted
    ci['secret'] = secret
    interview_status = docassemble.base.parse.InterviewStatus(current_info=ci)
    if not persistent:
        interview_status.checkin = True
    old_language = docassemble.base.functions.get_language()
    try:
        interview.assemble(user_dict, interview_status)
    except DAErrorMissingVariable as err:
        steps += 1
        save_user_dict(session_id, user_dict, yaml_filename, secret=secret, encrypt=is_encrypted, changed=True, steps=steps)
        release_lock(session_id, yaml_filename)
        docassemble.base.functions.set_language(old_language)
        return ('', 204)
    except Exception as e:
        release_lock(session_id, yaml_filename)
        docassemble.base.functions.set_language(old_language)
        return jsonify_with_status("api_session_action: failure to assemble interview: " + e.__class__.__name__ + ": " + str(e), 400)
    docassemble.base.functions.set_language(old_language)
    steps += 1
    save_user_dict(session_id, user_dict, yaml_filename, secret=secret, encrypt=is_encrypted, changed=True, steps=steps)
    release_lock(session_id, yaml_filename)
    if interview_status.question.question_type == "response":
        if hasattr(interview_status.question, 'all_variables'):
            if hasattr(interview_status.question, 'include_internal'):
                include_internal = interview_status.question.include_internal
            else:
                include_internal = False
            response_to_send = make_response(docassemble.base.functions.dict_as_json(user_dict, include_internal=include_internal).encode('utf-8'), '200 OK')
        elif hasattr(interview_status.question, 'binaryresponse'):
            response_to_send = make_response(interview_status.question.binaryresponse, '200 OK')
        else:
            response_to_send = make_response(interview_status.questionText.encode('utf-8'), '200 OK')
        response_to_send.headers['Content-Type'] = interview_status.extras['content_type']
        return response_to_send
    if interview_status.question.question_type == "sendfile":
        if interview_status.question.response_file is not None:
            the_path = interview_status.question.response_file.path()
        else:
            return jsonify_with_status("Could not send file because the response was None", 404)
        if not os.path.isfile(the_path):
            return jsonify_with_status("Could not send file because " + str(the_path) + " not found", 404)
        response_to_send = send_file(the_path, mimetype=interview_status.extras['content_type'])
        response_to_send.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
        return response_to_send
    return ('', 204)

@app.route('/api/login_url', methods=['POST'])
@csrf.exempt
@cross_origin(origins='*', methods=['POST', 'HEAD'], automatic_options=True)
def api_login_url():
    if not api_verify(request, roles=['admin']):
        return jsonify_with_status("Access denied.", 403)
    post_data = request.get_json(silent=True)
    if post_data is None:
        post_data = request.form.copy()
    username = post_data.get('username', None)
    password = post_data.get('password', None)
    if username is None or password is None:
        return jsonify_with_status("A username and password must be supplied", 400)
    try:
        secret = get_secret(str(username), str(password))
    except Exception as err:
        return jsonify_with_status(str(err), 403)
    try:
        expire = int(post_data.get('expire', 15))
        assert expire > 0
    except:
        return jsonify_with_status("Invalid number of seconds.", 400)
    if 'url_args' in post_data:
        if isinstance(post_data['url_args'], dict):
            url_args = post_data['url_args']
        else:
            try:
                url_args = json.loads(post_data['url_args'])
                assert isinstance(url_args, dict)
            except:
                return jsonify_with_status("Malformed URL arguments", 400)
    else:
        url_args = {}
    username = re.sub(r'\%', '', username)
    user = db.session.execute(select(UserModel).where(UserModel.email.ilike(username))).scalar()
    if user is None:
        return jsonify_with_status("Username not known", 403)
    info = dict(user_id=user.id, secret=secret)
    del user
    if 'next' in post_data:
        try:
            path = get_url_from_file_reference(post_data['next'])
            assert isinstance(path, str)
            assert not path.startswith('javascript')
        except:
            return jsonify_with_status("Unknown path for next", 400)
    for key in ['i', 'next', 'session']:
        if key in post_data:
            info[key] = post_data[key]
    if len(url_args) > 0:
        info['url_args'] = url_args
    if 'i' in info:
        if 'session' in info:
            try:
                steps, user_dict, is_encrypted = fetch_user_dict(info['session'], info['i'], secret=secret)
                info['encrypted'] = is_encrypted
            except:
                return jsonify_with_status("Could not decrypt dictionary", 400)
        elif true_or_false(post_data.get('resume_existing', False)) or daconfig.get('auto login resume existing', False):
            interviews = user_interviews(user_id=info['user_id'], secret=secret, exclude_invalid=True, filename=info['i'], include_dict=True)[0]
            if len(interviews) > 0:
                info['session'] = interviews[0]['session']
                info['encrypted'] = interviews[0]['encrypted']
            del interviews
    encryption_key = random_string(16)
    encrypted_text = encrypt_dictionary(info, encryption_key)
    while True:
        code = random_string(24)
        the_key = 'da:auto_login:' + code
        if r.get(the_key) is None:
            break
    pipe = r.pipeline()
    pipe.set(the_key, encrypted_text)
    pipe.expire(the_key, expire)
    pipe.execute()
    return jsonify(url_for('auto_login', key=encryption_key + code, _external=True))

@app.route('/api/list', methods=['GET'])
@cross_origin(origins='*', methods=['GET', 'HEAD'], automatic_options=True)
def api_list():
    if not api_verify(request):
        return jsonify_with_status("Access denied.", 403)
    return jsonify(interview_menu(absolute_urls=true_or_false(request.args.get('absolute_urls', True)), tag=request.args.get('tag', None)))

@app.route('/api/user/interviews', methods=['GET', 'DELETE'])
@csrf.exempt
@cross_origin(origins='*', methods=['GET', 'DELETE', 'HEAD'], automatic_options=True)
def api_user_interviews():
    if not api_verify(request):
        return jsonify_with_status("Access denied.", 403)
    filename = request.args.get('i', None)
    session = request.args.get('session', None)
    tag = request.args.get('tag', None)
    secret = request.args.get('secret', None)
    if secret is not None:
        secret = str(secret)
    include_dict = true_or_false(request.args.get('include_dictionary', False))
    next_id_code = request.args.get('next_id', None)
    if next_id_code:
        try:
            start_id = int(from_safeid(next_id_code))
            assert start_id >= 0
        except:
            start_id = None
    else:
        start_id = None
    if request.method == 'GET':
        try:
            (the_list, start_id) = user_interviews(user_id=current_user.id, secret=secret, filename=filename, session=session, exclude_invalid=False, tag=tag, include_dict=include_dict, start_id=start_id)
        except:
            return jsonify_with_status("Error reading interview list.", 400)
        if start_id is None:
            next_id = None
        else:
            next_id = safeid(str(start_id))
        return jsonify(dict(next_id=next_id, items=docassemble.base.functions.safe_json(the_list)))
    if request.method == 'DELETE':
        start_id = None
        while True:
            try:
                (the_list, start_id) = user_interviews(user_id=current_user.id, filename=filename, session=session, exclude_invalid=False, tag=tag, include_dict=False, start_id=start_id)
            except:
                return jsonify_with_status("Error reading interview list.", 400)
            for info in the_list:
                user_interviews(user_id=info['user_id'], action='delete', filename=info['filename'], session=info['session'])
            if start_id is None:
                break
        return ('', 204)
    return ('', 204)

@app.route('/api/interviews', methods=['GET', 'DELETE'])
@csrf.exempt
@cross_origin(origins='*', methods=['GET', 'DELETE', 'HEAD'], automatic_options=True)
def api_interviews():
    if not api_verify(request, roles=['admin', 'advocate']):
        return jsonify_with_status("Access denied.", 403)
    filename = request.args.get('i', None)
    session = request.args.get('session', None)
    tag = request.args.get('tag', None)
    secret = request.args.get('secret', None)
    if secret is not None:
        secret = str(secret)
    include_dict = true_or_false(request.args.get('include_dictionary', False))
    next_id_code = request.args.get('next_id', None)
    if next_id_code:
        try:
            start_id = int(from_safeid(next_id_code))
            assert start_id >= 0
        except:
            start_id = None
    else:
        start_id = None
    if request.method == 'GET':
        try:
            (the_list, start_id) = user_interviews(secret=secret, filename=filename, session=session, exclude_invalid=False, tag=tag, include_dict=include_dict, start_id=start_id)
        except Exception as err:
            return jsonify_with_status("Error reading interview list: " + str(err), 400)
        if start_id is None:
            next_id = None
        else:
            next_id = safeid(str(start_id))
        return jsonify(dict(next_id=next_id, items=docassemble.base.functions.safe_json(the_list)))
    if request.method == 'DELETE':
        start_id = None
        while True:
            try:
                (the_list, start_id) = user_interviews(filename=filename, session=session, exclude_invalid=False, tag=tag, include_dict=False, start_id=start_id)
            except:
                return jsonify_with_status("Error reading interview list.", 400)
            for info in the_list:
                if info['user_id'] is not None:
                    user_interviews(user_id=info['user_id'], action='delete', filename=info['filename'], session=info['session'])
                else:
                    user_interviews(temp_user_id=info['temp_user_id'], action='delete', filename=info['filename'], session=info['session'])
            if start_id is None:
                break
        return ('', 204)
    return ('', 204)

def jsonify_task(result):
    while True:
        code = random_string(24)
        the_key = 'da:install_status:' + code
        if r.get(the_key) is None:
            break
    pipe = r.pipeline()
    pipe.set(the_key, json.dumps({'id': result.id, 'server_start_time': START_TIME}))
    pipe.expire(the_key, 3600)
    pipe.execute()
    return jsonify({'task_id': code})

def jsonify_restart_task():
    while True:
        code = random_string(24)
        the_key = 'da:restart_status:' + code
        if r.get(the_key) is None:
            break
    pipe = r.pipeline()
    pipe.set(the_key, json.dumps({'server_start_time': START_TIME}))
    pipe.expire(the_key, 3600)
    pipe.execute()
    return jsonify({'task_id': code})

def should_run_create(package_name):
    if package_name in ('docassemble.base', 'docassemble.webapp', 'docassemble.demo', 'docassemble'):
        return True
    return False

@app.route('/api/package', methods=['GET', 'POST', 'DELETE'])
@csrf.exempt
@cross_origin(origins='*', methods=['GET', 'POST', 'DELETE', 'HEAD'], automatic_options=True)
def api_package():
    if not api_verify(request, roles=['admin', 'developer']):
        return jsonify_with_status("Access denied.", 403)
    if request.method == 'GET':
        package_list, package_auth = get_package_info(exclude_core=True)
        packages = []
        for package in package_list:
            if not package.package.active:
                continue
            item = {'name': package.package.name, 'type': package.package.type, 'can_update': package.can_update, 'can_uninstall': package.can_uninstall}
            if package.package.packageversion:
                item['version'] = package.package.packageversion
            if package.package.giturl:
                item['git_url'] = package.package.giturl
            if package.package.gitbranch:
                item['branch'] = package.package.gitbranch
            if package.package.upload:
                item['zip_file_number'] = package.package.upload
            packages.append(item)
        return jsonify(packages)
    if request.method == 'DELETE':
        target = request.args.get('package', None)
        do_restart = true_or_false(request.args.get('restart', True))
        if target is None:
            return jsonify_with_status("Missing package name.", 400)
        package_list, package_auth = get_package_info()
        the_package = None
        for package in package_list:
            if package.package.name == target:
                the_package = package
                break
        if the_package is None:
            return jsonify_with_status("Package not found.", 400)
        if not the_package.can_uninstall:
            return jsonify_with_status("You are not allowed to uninstall that package.", 400)
        uninstall_package(target)
        if do_restart:
            result = docassemble.webapp.worker.update_packages.apply_async(link=docassemble.webapp.worker.reset_server.s(run_create=should_run_create(target)))
        else:
            result = docassemble.webapp.worker.update_packages.delay(restart=False)
        return jsonify_task(result)
    if request.method == 'POST':
        post_data = request.get_json(silent=True)
        if post_data is None:
            post_data = request.form.copy()
        do_restart = true_or_false(post_data.get('restart', True))
        num_commands = 0
        if 'update' in post_data:
            num_commands += 1
        if 'github_url' in post_data:
            num_commands += 1
        if 'pip' in post_data:
            num_commands += 1
        if 'zip' in request.files:
            num_commands += 1
        if num_commands == 0:
            return jsonify_with_status("No instructions provided.", 400)
        if num_commands > 1:
            return jsonify_with_status("Only one package can be installed or updated at a time.", 400)
        if 'update' in post_data:
            target = post_data['update']
            package_list, package_auth = get_package_info()
            the_package = None
            for package in package_list:
                if package.package.name == target:
                    the_package = package
                    break
            if the_package is None:
                return jsonify_with_status("Package not found.", 400)
            if not the_package.can_update:
                return jsonify_with_status("You are not allowed to update that package.", 400)
            existing_package = db.session.execute(select(Package).filter_by(name=target, active=True).order_by(Package.id.desc())).scalar()
            if existing_package is not None:
                if existing_package.type == 'git' and existing_package.giturl is not None:
                    if existing_package.gitbranch:
                        install_git_package(target, existing_package.giturl, existing_package.gitbranch)
                    else:
                        install_git_package(target, existing_package.giturl, get_master_branch(existing_package.giturl))
                elif existing_package.type == 'pip':
                    if existing_package.name == 'docassemble.webapp' and existing_package.limitation:
                        existing_package.limitation = None
                    install_pip_package(existing_package.name, existing_package.limitation)
            db.session.commit()
            if do_restart:
                result = docassemble.webapp.worker.update_packages.apply_async(link=docassemble.webapp.worker.reset_server.s(run_create=should_run_create(target)))
            else:
                result = docassemble.webapp.worker.update_packages.delay(restart=False)
            return jsonify_task(result)
        if 'github_url' in post_data:
            github_url = post_data['github_url'].rstrip('/')
            branch = post_data.get('branch', None)
            if branch is None:
                branch = get_master_branch(github_url)
            packagename = re.sub(r'/*$', '', github_url)
            packagename = re.sub(r'^git+', '', packagename)
            packagename = re.sub(r'#.*', '', packagename)
            packagename = re.sub(r'\.git$', '', packagename)
            packagename = re.sub(r'.*/', '', packagename)
            packagename = re.sub(r'^docassemble-', 'docassemble.', packagename)
            if user_can_edit_package(giturl=github_url) and user_can_edit_package(pkgname=packagename):
                install_git_package(packagename, github_url, branch)
                if do_restart:
                    result = docassemble.webapp.worker.update_packages.apply_async(link=docassemble.webapp.worker.reset_server.s(run_create=should_run_create(packagename)))
                else:
                    result = docassemble.webapp.worker.update_packages.delay(restart=False)
                return jsonify_task(result)
            else:
                jsonify_with_status("You do not have permission to install that package.", 403)
        if 'pip' in post_data:
            m = re.match(r'([^>=<]+)([>=<]+.+)', post_data['pip'])
            if m:
                packagename = m.group(1)
                limitation = m.group(2)
            else:
                packagename = post_data['pip']
                limitation = None
            packagename = re.sub(r'[^A-Za-z0-9\_\-\.]', '', packagename)
            if user_can_edit_package(pkgname=packagename):
                install_pip_package(packagename, limitation)
                if do_restart:
                    result = docassemble.webapp.worker.update_packages.apply_async(link=docassemble.webapp.worker.reset_server.s(run_create=should_run_create(packagename)))
                else:
                    result = docassemble.webapp.worker.update_packages.delay(restart=False)
                return jsonify_task(result)
            else:
                return jsonify_with_status("You do not have permission to install that package.", 403)
        if 'zip' in request.files and request.files['zip'].filename:
            try:
                the_file = request.files['zip']
                filename = secure_filename(the_file.filename)
                file_number = get_new_file_number(docassemble.base.functions.get_uid(), filename)
                saved_file = SavedFile(file_number, extension='zip', fix=True, should_not_exist=True)
                file_set_attributes(file_number, private=False, persistent=True)
                zippath = saved_file.path
                the_file.save(zippath)
                saved_file.save()
                saved_file.finalize()
                pkgname = get_package_name_from_zip(zippath)
                if user_can_edit_package(pkgname=pkgname):
                    install_zip_package(pkgname, file_number)
                    if do_restart:
                        result = docassemble.webapp.worker.update_packages.apply_async(link=docassemble.webapp.worker.reset_server.s(run_create=should_run_create(pkgname)))
                    else:
                        result = docassemble.webapp.worker.update_packages.delay(restart=False)
                    return jsonify_task(result)
                return jsonify_with_status("You do not have permission to install that package.", 403)
            except:
                return jsonify_with_status("There was an error when installing that package.", 400)

@app.route('/api/package_update_status', methods=['GET'])
@csrf.exempt
@cross_origin(origins='*', methods=['GET', 'HEAD'], automatic_options=True)
def api_package_update_status():
    if not api_verify(request, roles=['admin', 'developer']):
        return jsonify_with_status("Access denied.", 403)
    code = request.args.get('task_id', None)
    if code is None:
        return jsonify_with_status("Missing task_id", 400)
    the_key = 'da:install_status:' + str(code)
    task_data = r.get(the_key)
    if task_data is None:
        return jsonify({'status': 'unknown'})
    task_info = json.loads(task_data.decode())
    result = docassemble.webapp.worker.workerapp.AsyncResult(id=task_info['id'])
    if result.ready():
        the_result = result.get()
        if isinstance(the_result, ReturnValue):
            if the_result.ok:
                if the_result.restart and START_TIME <= task_info['server_start_time']:
                    return jsonify(status='working')
                r.delete(the_key)
                return jsonify(status='completed', ok=True, log=summarize_results(the_result.results, the_result.logmessages, html=False))
            if hasattr(the_result, 'error_message'):
                r.delete(the_key)
                return jsonify(status='completed', ok=False, error_message=str(the_result.error_message))
            if hasattr(the_result, 'results') and hasattr(the_result, 'logmessages'):
                r.delete(the_key)
                return jsonify(status='completed', ok=False, error_message=summarize_results(the_result.results, the_result.logmessages, html=False))
            r.expire(the_key, 30)
            return jsonify(status='completed', ok=False, error_message=str("No error message.  Result is " + str(the_result)))
        r.expire(the_key, 30)
        return jsonify(status='completed', ok=False, error_message=str(the_result))
    return jsonify(status='working')

@app.route('/api/temp_url', methods=['GET'])
@csrf.exempt
@cross_origin(origins='*', methods=['GET', 'HEAD'], automatic_options=True)
def api_temporary_redirect():
    if not api_verify(request):
        return jsonify_with_status("Access denied.", 403)
    url = request.args.get('url', None)
    if url is None:
        return jsonify_with_status("No url supplied.", 400)
    try:
        one_time = bool(int(request.args.get('one_time', 0)))
    except:
        one_time = False
    try:
        expire = int(request.args.get('expire', 3600))
        assert expire > 0
    except:
        return jsonify_with_status("Invalid number of seconds.", 400)
    return jsonify(docassemble.base.functions.temp_redirect(url, expire, False, one_time))

@app.route('/api/resume_url', methods=['POST'])
@csrf.exempt
@cross_origin(origins='*', methods=['POST', 'HEAD'], automatic_options=True)
def api_resume_url():
    if not api_verify(request):
        return jsonify_with_status("Access denied.", 403)
    post_data = request.get_json(silent=True)
    if post_data is None:
        post_data = request.form.copy()
    filename = post_data.get('i', None)
    if filename is None:
        return jsonify_with_status("No filename supplied.", 400)
    session = post_data.get('session', None)
    if 'url_args' in post_data:
        if isinstance(post_data['url_args'], dict):
            url_args = post_data['url_args']
        else:
            try:
                url_args = json.loads(post_data['url_args'])
                assert isinstance(url_args, dict)
            except:
                return jsonify_with_status("Malformed URL arguments", 400)
    else:
        url_args = {}
    try:
        one_time = bool(int(post_data.get('one_time', 0)))
    except:
        one_time = False
    try:
        expire = int(post_data.get('expire', 3600))
        assert expire > 0
    except:
        return jsonify_with_status("Invalid number of seconds.", 400)
    info = dict(i=filename)
    if session:
        info['session'] = session
    if one_time:
        info['once'] = True
    while True:
        code = random_string(32)
        the_key = 'da:resume_interview:' + code
        if r.get(the_key) is None:
            break
    pipe = r.pipeline()
    pipe.set(the_key, json.dumps(info))
    pipe.expire(the_key, expire)
    pipe.execute()
    return jsonify(url_for('launch', c=code, _external=True))

@app.route('/api/clear_cache', methods=['POST'])
@csrf.exempt
@cross_origin(origins='*', methods=['POST', 'HEAD'], automatic_options=True)
def api_clear_cache():
    if not api_verify(request, roles=['admin', 'developer']):
        return jsonify_with_status("Access denied.", 403)
    for key in r.keys('da:interviewsource:*'):
        r.incr(key.decode())
    return ('', 204)

@app.route('/api/config', methods=['GET', 'POST', 'PATCH'])
@csrf.exempt
@cross_origin(origins='*', methods=['GET', 'POST', 'PATCH', 'HEAD'], automatic_options=True)
def api_config():
    if not api_verify(request, roles=['admin']):
        return jsonify_with_status("Access denied.", 403)
    if request.method == 'GET':
        try:
            with open(daconfig['config file'], 'r', encoding='utf-8') as fp:
                content = fp.read()
            data = yaml.load(content, Loader=yaml.FullLoader)
        except:
            return jsonify_with_status("Could not parse Configuration.", 400)
        return jsonify(data)
    if request.method == 'POST':
        post_data = request.get_json(silent=True)
        if post_data is None:
            post_data = request.form.copy()
        if 'config' not in post_data:
            return jsonify_with_status("Configuration not supplied.", 400)
        if isinstance(post_data['config'], dict):
            data = post_data['config']
        else:
            try:
                data = json.loads(post_data['config'])
            except:
                return jsonify_with_status("Configuration was not valid JSON.", 400)
        yaml_data = ruamel.yaml.safe_dump(data, default_flow_style=False, default_style = '"', allow_unicode=True, width=10000)
        if cloud is not None:
            key = cloud.get_key('config.yml')
            key.set_contents_from_string(yaml_data)
        with open(daconfig['config file'], 'w', encoding='utf-8') as fp:
            fp.write(yaml_data)
        return_val = jsonify_restart_task()
        restart_all()
        return return_val
    if request.method == 'PATCH':
        try:
            with open(daconfig['config file'], 'r', encoding='utf-8') as fp:
                content = fp.read()
            data = yaml.load(content, Loader=yaml.FullLoader)
        except:
            return jsonify_with_status("Could not parse Configuration.", 400)
        patch_data = request.get_json(silent=True)
        if patch_data is None:
            using_json = False
            patch_data = request.form.copy()
        else:
            using_json = True
        if 'config_changes' not in patch_data:
            return jsonify_with_status("Configuration changes not supplied.", 400)
        if isinstance(patch_data['config_changes'], dict):
            new_data = patch_data['config_changes']
        else:
            try:
                new_data = json.loads(patch_data['config_changes'])
            except:
                return jsonify_with_status("Configuration changes were not valid JSON.", 400)
        data.update(new_data)
        yaml_data = ruamel.yaml.safe_dump(data, default_flow_style=False, default_style = '"', allow_unicode=True, width=10000)
        if cloud is not None:
            key = cloud.get_key('config.yml')
            key.set_contents_from_string(yaml_data)
        with open(daconfig['config file'], 'w', encoding='utf-8') as fp:
            fp.write(yaml_data)
        return_val = jsonify_restart_task()
        restart_all()
        return return_val

@app.route('/api/playground_pull', methods=['GET', 'POST'])
@csrf.exempt
@cross_origin(origins='*', methods=['POST', 'HEAD'], automatic_options=True)
def api_playground_pull():
    if not api_verify(request, roles=['admin', 'developer']):
        return jsonify_with_status("Access denied.", 403)
    post_data = request.get_json(silent=True)
    if post_data is None:
        post_data = request.form.copy()
    do_restart = true_or_false(post_data.get('restart', True))
    current_project = post_data.get('project', 'default')
    try:
        if current_user.has_role('admin'):
            user_id = int(post_data.get('user_id', current_user.id))
        else:
            if 'user_id' in post_data:
                assert int(post_data['user_id']) == current_user.id
            user_id = current_user.id
    except:
        return jsonify_with_status("Invalid user_id.", 400)
    if current_project != 'default' and current_project not in get_list_of_projects(user_id):
        return jsonify_with_status("Invalid project.", 400)
    docassemble.base.functions.this_thread.current_info['user'] = dict(is_anonymous=False, theid=user_id)
    if app.config['USE_GITHUB']:
        github_auth = r.get('da:using_github:userid:' + str(current_user.id))
        can_publish_to_github = bool(github_auth is not None)
    else:
        can_publish_to_github = None
    github_url = None
    branch = None
    pypi_package = None
    if 'github_url' in post_data:
        github_url = post_data['github_url'].rstrip('/')
        branch = post_data.get('branch', None)
        if branch is None:
            branch = get_master_branch(github_url)
        packagename = re.sub(r'/*$', '', github_url)
        packagename = re.sub(r'^git+', '', packagename)
        packagename = re.sub(r'#.*', '', packagename)
        packagename = re.sub(r'\.git$', '', packagename)
        packagename = re.sub(r'.*/', '', packagename)
        packagename = re.sub(r'^docassemble-', 'docassemble.', packagename)
    elif 'pip' in post_data:
        m = re.match(r'([^>=<]+)([>=<]+.+)', post_data['pip'])
        if m:
            pypi_package = m.group(1)
            limitation = m.group(2)
        else:
            pypi_package = post_data['pip']
            limitation = None
        packagename = re.sub(r'[^A-Za-z0-9\_\-\.]', '', pypi_package)
    else:
        return jsonify_with_status("Either github_url or pip is required.", 400)
    area = {}
    area_sec = dict(templates='playgroundtemplate', static='playgroundstatic', sources='playgroundsources', questions='playground')
    for sec in ('playground', 'playgroundpackages', 'playgroundtemplate', 'playgroundstatic', 'playgroundsources', 'playgroundmodules'):
        area[sec] = SavedFile(user_id, fix=True, section=sec)
    result = do_playground_pull(area, current_project, github_url=github_url, branch=branch, pypi_package=pypi_package, can_publish_to_github=can_publish_to_github, github_email=current_user.email)
    if result['action'] in ('error', 'fail'):
        return jsonify_with_status("Pull process encountered an error: " + result['message'], 400)
    if result['action'] == 'finished':
        if result['need_to_restart'] and do_restart:
            return_val = jsonify_restart_task()
            restart_all()
            return return_val
    return ('', 204)

@app.route('/api/restart', methods=['POST'])
@csrf.exempt
@cross_origin(origins='*', methods=['POST', 'HEAD'], automatic_options=True)
def api_restart():
    if not api_verify(request, roles=['admin', 'developer']):
        return jsonify_with_status("Access denied.", 403)
    return_val = jsonify_restart_task()
    restart_all()
    return return_val

@app.route('/api/restart_status', methods=['GET'])
@csrf.exempt
@cross_origin(origins='*', methods=['GET', 'HEAD'], automatic_options=True)
def api_restart_status():
    if not api_verify(request, roles=['admin', 'developer']):
        return jsonify_with_status("Access denied.", 403)
    code = request.args.get('task_id', None)
    if code is None:
        return jsonify_with_status("Missing task_id", 400)
    the_key = 'da:restart_status:' + str(code)
    task_data = r.get(the_key)
    if task_data is None:
        return jsonify(status= 'unknown')
    task_info = json.loads(task_data.decode())
    if START_TIME <= task_info['server_start_time']:
        return jsonify(status='working')
    r.expire(the_key, 30)
    return jsonify(status='completed')

@app.route('/api/playground_install', methods=['POST'])
@csrf.exempt
@cross_origin(origins='*', methods=['POST', 'HEAD'], automatic_options=True)
def api_playground_install():
    if not api_verify(request, roles=['admin', 'developer']):
        return jsonify_with_status("Access denied.", 403)
    post_data = request.get_json(silent=True)
    if post_data is None:
        post_data = request.form.copy()
    do_restart = true_or_false(post_data.get('restart', True))
    current_project = post_data.get('project', 'default')
    try:
        if current_user.has_role('admin'):
            user_id = int(post_data.get('user_id', current_user.id))
        else:
            if 'user_id' in post_data:
                assert int(post_data['user_id']) == current_user.id
            user_id = current_user.id
    except:
        return jsonify_with_status("Invalid user_id.", 400)
    if current_project != 'default' and current_project not in get_list_of_projects(user_id):
        return jsonify_with_status("Invalid project.", 400)
    docassemble.base.functions.this_thread.current_info['user'] = dict(is_anonymous=False, theid=user_id)
    found = False
    expected_name = 'unknown'
    need_to_restart = False
    area = {}
    area_sec = dict(templates='playgroundtemplate', static='playgroundstatic', sources='playgroundsources', questions='playground')
    for sec in ('playground', 'playgroundpackages', 'playgroundtemplate', 'playgroundstatic', 'playgroundsources', 'playgroundmodules'):
        area[sec] = SavedFile(user_id, fix=True, section=sec)
    try:
        for filekey in request.files:
            the_files = request.files.getlist(filekey)
            if not the_files:
                continue
            for up_file in the_files:
                found = True
                zippath = tempfile.NamedTemporaryFile(mode="wb", prefix='datemp', suffix=".zip", delete=False)
                up_file.save(zippath)
                up_file.close()
                zippath.close()
                with zipfile.ZipFile(zippath.name, mode='r') as zf:
                    readme_text = ''
                    setup_py = ''
                    extracted = {}
                    data_files = dict(templates=[], static=[], sources=[], interviews=[], modules=[], questions=[])
                    has_docassemble_dir = set()
                    has_setup_file = set()
                    for zinfo in zf.infolist():
                        if zinfo.is_dir():
                            if zinfo.filename.endswith('/docassemble/'):
                                has_docassemble_dir.add(re.sub(r'/docassemble/$', '', zinfo.filename))
                            if zinfo.filename == 'docassemble/':
                                has_docassemble_dir.add('')
                        elif zinfo.filename.endswith('/setup.py'):
                            (directory, filename) = os.path.split(zinfo.filename)
                            has_setup_file.add(directory)
                        elif zinfo.filename == 'setup.py':
                            has_setup_file.add('')
                    root_dir = None
                    for directory in has_docassemble_dir.union(has_setup_file):
                        if root_dir is None or len(directory) < len(root_dir):
                            root_dir = directory
                    if root_dir is None:
                        return jsonify_with_status("Not a docassemble package.", 400)
                    for zinfo in zf.infolist():
                        if zinfo.filename.endswith('/'):
                            continue
                        (directory, filename) = os.path.split(zinfo.filename)
                        if filename.startswith('#') or filename.endswith('~'):
                            continue
                        dirparts = splitall(directory)
                        if '.git' in dirparts:
                            continue
                        levels = re.findall(r'/', directory)
                        time_tuple = zinfo.date_time
                        the_time = time.mktime(datetime.datetime(*time_tuple).timetuple())
                        for sec in ('templates', 'static', 'sources', 'questions'):
                            if directory.endswith('data/' + sec) and filename != 'README.md':
                                data_files[sec].append(filename)
                                target_filename = os.path.join(directory_for(area[area_sec[sec]], current_project), filename)
                                with zf.open(zinfo) as source_fp, open(target_filename, 'wb') as target_fp:
                                    shutil.copyfileobj(source_fp, target_fp)
                                os.utime(target_filename, (the_time, the_time))
                        if filename == 'README.md' and directory == root_dir:
                            with zf.open(zinfo) as f:
                                the_file_obj = TextIOWrapper(f, encoding='utf8')
                                readme_text = the_file_obj.read()
                        if filename == 'setup.py' and directory == root_dir:
                            with zf.open(zinfo) as f:
                                the_file_obj = TextIOWrapper(f, encoding='utf8')
                                setup_py = the_file_obj.read()
                        elif len(levels) >= 2 and filename.endswith('.py') and filename != '__init__.py' and 'tests' not in dirparts and 'data' not in dirparts:
                            need_to_restart = True
                            data_files['modules'].append(filename)
                            target_filename = os.path.join(directory_for(area['playgroundmodules'], current_project), filename)
                            with zf.open(zinfo) as source_fp, open(target_filename, 'wb') as target_fp:
                                shutil.copyfileobj(source_fp, target_fp)
                                os.utime(target_filename, (the_time, the_time))
                    setup_py = re.sub(r'.*setup\(', '', setup_py, flags=re.DOTALL)
                    for line in setup_py.splitlines():
                        m = re.search(r"^ *([a-z_]+) *= *\(?'(.*)'", line)
                        if m:
                            extracted[m.group(1)] = m.group(2)
                        m = re.search(r'^ *([a-z_]+) *= *\(?"(.*)"', line)
                        if m:
                            extracted[m.group(1)] = m.group(2)
                        m = re.search(r'^ *([a-z_]+) *= *\[(.*)\]', line)
                        if m:
                            the_list = []
                            for item in re.split(r', *', m.group(2)):
                                inner_item = re.sub(r"'$", '', item)
                                inner_item = re.sub(r"^'", '', inner_item)
                                inner_item = re.sub(r'"+$', '', inner_item)
                                inner_item = re.sub(r'^"+', '', inner_item)
                                the_list.append(inner_item)
                            extracted[m.group(1)] = the_list
                    info_dict = dict(readme=readme_text, interview_files=data_files['questions'], sources_files=data_files['sources'], static_files=data_files['static'], module_files=data_files['modules'], template_files=data_files['templates'], dependencies=[z for z in map(lambda y: re.sub(r'[\>\<\=].*', '', y), extracted.get('install_requires', []))], description=extracted.get('description', ''), author_name=extracted.get('author', ''), author_email=extracted.get('author_email', ''), license=extracted.get('license', ''), url=extracted.get('url', ''), version=extracted.get('version', ''))

                    info_dict['dependencies'] = [x for x in [z for z in map(lambda y: re.sub(r'[\>\<\=].*', '', y), info_dict['dependencies'])] if x not in ('docassemble', 'docassemble.base', 'docassemble.webapp')]
                    package_name = re.sub(r'^docassemble\.', '', extracted.get('name', expected_name))
                    with open(os.path.join(directory_for(area['playgroundpackages'], current_project), 'docassemble.' + package_name), 'w', encoding='utf-8') as fp:
                        the_yaml = yaml.safe_dump(info_dict, default_flow_style=False, default_style='|')
                        fp.write(str(the_yaml))
                    for key in r.keys('da:interviewsource:docassemble.playground' + str(current_user.id) + project_name(current_project) + ':*'):
                        r.incr(key.decode())
                    for sec in area:
                        area[sec].finalize()
                    the_file = package_name
                zippath.close()
    except Exception as err:
        logmessage("api_playground_install: " + err.__class__.__name__ + ": " + str(err))
        return jsonify_with_status("Error installing packages.", 400)
    if not found:
        return jsonify_with_status("No package found.", 400)
    for key in r.keys('da:interviewsource:docassemble.playground' + str(user_id) + project_name(current_project) + ':*'):
        r.incr(key.decode())
    if do_restart and need_to_restart:
        return_val = jsonify_restart_task()
        restart_all()
        return return_val
    return ('', 204)

@app.route('/api/playground/project', methods=['GET', 'POST', 'DELETE'])
@csrf.exempt
@cross_origin(origins='*', methods=['GET', 'POST', 'DELETE', 'HEAD'], automatic_options=True)
def api_playground_projects():
    if not api_verify(request, roles=['admin', 'developer']):
        return jsonify_with_status("Access denied.", 403)
    if request.method in ('GET', 'DELETE'):
        try:
            if current_user.has_role('admin'):
                user_id = int(request.args.get('user_id', current_user.id))
            else:
                if 'user_id' in request.args:
                    assert int(request.args['user_id']) == current_user.id
                user_id = current_user.id
        except:
            return jsonify_with_status("Invalid user_id.", 400)
    if request.method == 'GET':
        return jsonify(get_list_of_projects(user_id))
    if request.method == 'DELETE':
        if 'project' not in request.args:
            return jsonify_with_status("Project not provided.", 400)
        project = request.args['project']
        if project not in get_list_of_projects(user_id) or project == 'default':
            return jsonify_with_status("Invalid project.", 400)
        delete_project(user_id, project)
        return ('', 204)
    if request.method == 'POST':
        post_data = request.get_json(silent=True)
        if post_data is None:
            post_data = request.form.copy()
        try:
            if current_user.has_role('admin'):
                user_id = int(post_data.get('user_id', current_user.id))
            else:
                if 'user_id' in post_data:
                    assert int(post_data['user_id']) == current_user.id
                user_id = current_user.id
        except:
            return jsonify_with_status("Invalid user_id.", 400)
        if 'project' not in post_data:
            return jsonify_with_status("Project not provided.", 400)
        project = post_data['project']
        if re.search('^[0-9]', project) or re.search('[^A-Za-z0-9]', project):
            return jsonify_with_status("Invalid project name.", 400)
        if project in get_list_of_projects(user_id) or project == 'default':
            return jsonify_with_status("Invalid project.", 400)
        create_project(user_id, project)
        return ('', 204)

@app.route('/api/playground', methods=['GET', 'POST', 'DELETE'])
@csrf.exempt
@cross_origin(origins='*', methods=['GET', 'POST', 'DELETE', 'HEAD'], automatic_options=True)
def api_playground():
    if not api_verify(request, roles=['admin', 'developer']):
        return jsonify_with_status("Access denied.", 403)
    if request.method in ('GET', 'DELETE'):
        folder = request.args.get('folder', 'static')
        project = request.args.get('project', 'default')
        try:
            if current_user.has_role('admin'):
                user_id = int(request.args.get('user_id', current_user.id))
            else:
                if 'user_id' in request.args:
                    assert int(request.args['user_id']) == current_user.id
                user_id = current_user.id
        except:
            return jsonify_with_status("Invalid user_id.", 400)
    elif request.method == 'POST':
        post_data = request.get_json(silent=True)
        if post_data is None:
            post_data = request.form.copy()
        folder = post_data.get('folder', 'static')
        project = post_data.get('project', 'default')
        try:
            if current_user.has_role('admin'):
                user_id = int(post_data.get('user_id', current_user.id))
            else:
                if 'user_id' in post_data:
                    assert int(post_data['user_id']) == current_user.id
                user_id = current_user.id
        except:
            return jsonify_with_status("Invalid user_id.", 400)
    if request.method == 'DELETE':
        if 'filename' not in request.args:
            return jsonify_with_status("Missing filename.", 400)
    if folder not in ('questions', 'sources', 'static', 'templates', 'modules'):
        return jsonify_with_status("Invalid folder.", 400)
    if project != 'default' and project not in get_list_of_projects(user_id):
        return jsonify_with_status("Invalid project.", 400)
    if folder == 'questions':
        section = ''
    elif folder == 'templates':
        section = 'template'
    else:
        section = folder
    docassemble.base.functions.this_thread.current_info['user'] = dict(is_anonymous=False, theid=user_id)
    pg_section = PlaygroundSection(section=section, project=project)
    if request.method == 'GET':
        if 'filename' not in request.args:
            return jsonify(pg_section.file_list)
        the_filename = secure_filename_spaces_ok(request.args['filename'])
        if not pg_section.file_exists(the_filename):
            return jsonify_with_status("File not found", 404)
        response_to_send = send_file(pg_section.get_file(the_filename), mimetype=pg_section.get_mimetype(the_filename))
        response_to_send.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
        return response_to_send
    if request.method == 'DELETE':
        pg_section.delete_file(secure_filename_spaces_ok(request.args['filename']))
        if section == 'modules':
            return_val = jsonify_restart_task()
            restart_all()
            return return_val
        return ('', 204)
    if request.method == 'POST':
        found = False
        try:
            for filekey in request.files:
                the_files = request.files.getlist(filekey)
                if the_files:
                    for the_file in the_files:
                        filename = werkzeug.utils.secure_filename(the_file.filename)
                        temp_file = tempfile.NamedTemporaryFile(prefix="datemp", delete=False)
                        the_file.save(temp_file.name)
                        pg_section.copy_from(temp_file.name, filename=filename)
                        found = True
        except:
            return jsonify_with_status("Error saving file(s).", 400)
        if not found:
            return jsonify_with_status("No file found.", 400)
        for key in r.keys('da:interviewsource:docassemble.playground' + str(user_id) + project_name(project) + ':*'):
            r.incr(key.decode())
        if section == 'modules':
            return_val = jsonify_restart_task()
            restart_all()
            return return_val
        return ('', 204)
    return ('', 204)

@app.route('/api/convert_file', methods=['POST'])
@csrf.exempt
@cross_origin(origins='*', methods=['POST', 'HEAD'], automatic_options=True)
def api_convert_file():
    if not api_verify(request):
        return jsonify_with_status("Access denied.", 403)
    post_data = request.form.copy()
    to_format = post_data.get('format', 'md')
    if to_format not in 'md':
        return jsonify_with_status("Invalid output file format.", 400)
    for filekey in request.files:
        the_files = request.files.getlist(filekey)
        if the_files:
            for the_file in the_files:
                filename = werkzeug.utils.secure_filename(the_file.filename)
                extension, mimetype = get_ext_and_mimetype(filename)
                if mimetype and mimetype in convertible_mimetypes:
                    the_format = convertible_mimetypes[mimetype]
                elif extension and extension in convertible_extensions:
                    the_format = convertible_extensions[extension]
                else:
                    return jsonify_with_status("Invalid input file format.", 400)
                with tempfile.NamedTemporaryFile() as temp_file:
                    the_file.save(temp_file.name)
                    result = word_to_markdown(temp_file.name, the_format)
                    if result is None:
                        return jsonify_with_status("Unable to convert file.", 400)
                    with open(result.name, 'r', encoding='utf-8') as fp:
                        contents = fp.read()
                response = make_response(contents, 200)
                response.headers['Content-Type'] = 'text/plain'
                return response

def add_api_key(user_id, name, method, allowed):
    info = dict(constraints=allowed, method=method, name=name)
    success = False
    for attempt in range(10):
        api_key = random_alphanumeric(32)
        info['last_four'] = api_key[-4:]
        new_api_key = encrypt_api_key(api_key)
        if not len(r.keys('da:apikey:userid:*:key:' + new_api_key + ':info')):
            r.set('da:apikey:userid:' + str(user_id) + ':key:' + new_api_key + ':info', json.dumps(info))
            success = True
            break
    if not success:
        return None
    return api_key

def api_key_exists(user_id, api_key):
    rkeys = r.keys('da:apikey:userid:' + str(user_id) + ':key:' + encrypt_api_key(str(api_key)) + ':info')
    if len(rkeys) > 0:
        return True
    return False

def existing_api_names(user_id, except_for=None):
    result = []
    rkeys = r.keys('da:apikey:userid:' + str(user_id) + ':key:*:info')
    for key in rkeys:
        key = key.decode()
        if except_for is not None:
            api_key = re.sub(r'.*:key:([^:]+):.*', r'\1', key)
            if api_key == encrypt_api_key(except_for):
                continue
        try:
            info = json.loads(r.get(key).decode())
            result.append(info['name'])
        except:
            continue
    return result

def get_api_info(user_id, name=None, api_key=None):
    result = []
    rkeys = r.keys('da:apikey:userid:' + str(user_id) + ':key:*:info')
    if api_key is not None:
        api_key = encrypt_api_key(api_key)
    for key in rkeys:
        key = key.decode()
        try:
            info = json.loads(r.get(key).decode())
        except:
            logmessage("API information could not be unpacked.")
            continue
        if name is not None:
            if info['name'] == name:
                return info
        info['key'] = ('*' * 28) + info['last_four']
        this_key = re.sub(r'.*:key:([^:]+):.*', r'\1', key)
        if api_key is not None and this_key == api_key:
            return info
        if name is not None or api_key is not None:
            continue
        result.append(info)
    if name is not None or api_key is not None:
        return None
    return result

def delete_api_key(user_id, api_key):
    key = 'da:apikey:userid:' + str(user_id) + ':key:' + encrypt_api_key(api_key) + ':info'
    r.delete(key)

def update_api_key(user_id, api_key, name, method, allowed, add_to_allowed, remove_from_allowed):
    key = 'da:apikey:userid:' + str(user_id) + ':key:' + encrypt_api_key(api_key) + ':info'
    try:
        info = json.loads(r.get(key).decode())
    except:
        return False
    if name is not None:
        info['name'] = name
    if method is not None:
        if info['method'] != method:
            info['constraints'] = []
        info['method'] = method
    if allowed is not None:
        info['constraints'] = allowed
    if add_to_allowed is not None:
        if isinstance(add_to_allowed, list):
            info['constraints'].extend(add_to_allowed)
        elif isinstance(add_to_allowed, str):
            info['constraints'].append(add_to_allowed)
    if remove_from_allowed is not None:
        if isinstance(remove_from_allowed, list):
            to_remove = remove_from_allowed
        elif isinstance(remove_from_allowed, str):
            to_remove = [remove_from_allowed]
        else:
            to_remove = []
        for item in to_remove:
            if item in info['constraints']:
                info['constraints'].remove(item)
    r.set(key, json.dumps(info))
    return True

def do_api_user_api(user_id):
    if request.method == 'GET':
        name = request.args.get('name', None)
        api_key = request.args.get('api_key', None)
        try:
            result = get_api_info(user_id, name=name, api_key=api_key)
        except:
            return jsonify_with_status("Error accessing API information", 400)
        if (name is not None or api_key is not None) and result is None:
            return jsonify_with_status("No such API key could be found.", 404)
        return jsonify(result)
    if request.method == 'DELETE':
        api_key = request.args.get('api_key', None)
        if api_key is None:
            return jsonify_with_status("An API key must supplied", 400)
        try:
            delete_api_key(user_id, api_key)
        except:
            return jsonify_with_status("Error deleting API key", 400)
        return ('', 204)
    if request.method == 'POST':
        post_data = request.get_json(silent=True)
        if post_data is None:
            post_data = request.form.copy()
        name = post_data.get('name', None)
        method = post_data.get('method', 'none')
        if method not in ('ip', 'referer', 'none'):
            return jsonify_with_status("Invalid security method", 400)
        allowed = post_data.get('allowed', [])
        if isinstance(allowed, str):
            try:
                allowed = json.loads(allowed)
            except:
                return jsonify_with_status("Allowed sites list not a valid list", 400)
        if not isinstance(allowed, list):
            return jsonify_with_status("Allowed sites list not a valid list", 400)
        try:
            for item in allowed:
                assert isinstance(item, str)
        except:
            return jsonify_with_status("Allowed sites list not a valid list", 400)
        if name is None:
            return jsonify_with_status("A name must be supplied", 400)
        if name in existing_api_names(user_id):
            return jsonify_with_status("The given name already exists", 400)
        if len(name) > 255:
            return jsonify_with_status("The name is invalid", 400)
        new_api_key = add_api_key(user_id, name, method, allowed)
        if new_api_key is None:
            return jsonify_with_status("Error creating API key", 400)
        else:
            return jsonify(new_api_key)
    if request.method == 'PATCH':
        patch_data = request.get_json(silent=True)
        if patch_data is None:
            patch_data = request.form.copy()
        if current_user.id == user_id:
            api_key = patch_data.get('api_key', get_api_key())
        else:
            api_key = patch_data.get('api_key', None)
            if api_key is None:
                return jsonify_with_status("No API key given", 400)
        if not api_key_exists(user_id, api_key):
            return jsonify_with_status("The given API key cannot be modified", 400)
        name = patch_data.get('name', None)
        if name is not None:
            if name in existing_api_names(user_id, except_for=api_key):
                return jsonify_with_status("The given name already exists", 400)
            if len(name) > 255:
                return jsonify_with_status("The name is invalid", 400)
        method = patch_data.get('method', None)
        if method is not None:
            if method not in ('ip', 'referer', 'none'):
                return jsonify_with_status("Invalid security method", 400)
        allowed = patch_data.get('allowed', None)
        add_to_allowed = patch_data.get('add_to_allowed', None)
        if add_to_allowed is not None:
            if add_to_allowed.startswith('['):
                try:
                    add_to_allowed = json.loads(add_to_allowed)
                    for item in add_to_allowed:
                        assert isinstance(item, str)
                except:
                    return jsonify_with_status("add_to_allowed is not a valid list", 400)
        remove_from_allowed = patch_data.get('remove_from_allowed', None)
        if remove_from_allowed is not None:
            if remove_from_allowed.startswith('['):
                try:
                    remove_from_allowed = json.loads(remove_from_allowed)
                    for item in remove_from_allowed:
                        assert isinstance(item, str)
                except:
                    return jsonify_with_status("remove_from_allowed is not a valid list", 400)
        if allowed is not None:
            if isinstance(allowed, str):
                try:
                    allowed = json.loads(allowed)
                except:
                    return jsonify_with_status("Allowed sites list not a valid list", 400)
            if not isinstance(allowed, list):
                return jsonify_with_status("Allowed sites list not a valid list", 400)
            try:
                for item in allowed:
                    assert isinstance(item, str)
            except:
                return jsonify_with_status("Allowed sites list not a valid list", 400)
        result = update_api_key(user_id, api_key, name, method, allowed, add_to_allowed, remove_from_allowed)
        if not result:
            return jsonify_with_status("Error updating API key", 400)
        return ('', 204)

@app.route('/api/user/api', methods=['GET', 'POST', 'DELETE', 'PATCH'])
@csrf.exempt
@cross_origin(origins='*', methods=['GET', 'POST', 'DELETE', 'PATCH', 'HEAD'], automatic_options=True)
def api_user_api():
    if not api_verify(request):
        return jsonify_with_status("Access denied.", 403)
    return do_api_user_api(current_user.id)

@app.route('/api/user/<int:user_id>/api', methods=['GET', 'POST', 'DELETE', 'PATCH'])
@csrf.exempt
@cross_origin(origins='*', methods=['GET', 'POST', 'DELETE', 'PATCH', 'HEAD'], automatic_options=True)
def api_user_userid_api(user_id):
    if not api_verify(request):
        return jsonify_with_status("Access denied.", 403)
    try:
        user_id = int(user_id)
    except:
        return jsonify_with_status("User ID must be an integer.", 400)
    if not (current_user.id == user_id or current_user.has_role('admin')):
        return jsonify_with_status("Access denied.", 403)
    try:
        user_info = get_user_info(user_id=user_id)
    except Exception as err:
        return jsonify_with_status("Error obtaining user information: " + str(err), 400)
    if user_info is None:
        return jsonify_with_status("User not found.", 404)
    return do_api_user_api(user_id)

@app.route('/api/interview_data', methods=['GET'])
@csrf.exempt
@cross_origin(origins='*', methods=['GET', 'HEAD'], automatic_options=True)
def api_interview_data():
    if not api_verify(request, roles=['admin', 'developer']):
        return jsonify_with_status("Access denied.", 403)
    filename = request.args.get('i', None)
    if filename is None:
        return jsonify_with_status("No filename supplied.", 400)
    try:
        interview_source = docassemble.base.parse.interview_source_from_string(filename, testing=True)
    except Exception as err:
        return jsonify_with_status("Error finding interview: " + str(err), 400)
    try:
        interview = interview_source.get_interview()
    except Exception as err:
        return jsonify_with_status("Error finding interview: " + str(err), 400)
    device_id = docassemble.base.functions.this_thread.current_info['user']['device_id']
    interview_status = docassemble.base.parse.InterviewStatus(current_info=current_info(yaml=filename, req=request, action=None, device_id=device_id))
    m = re.search('docassemble.playground([0-9]+)([^:]*):', filename)
    if m:
        use_playground = bool(current_user.id == int(m.group(1)))
        if m.group(2) != '':
            current_project = m.group(2)
        else:
            current_project = 'default'
    else:
        use_playground = False
        current_project = 'default'
    variables_json, vocab_list = get_vars_in_use(interview, interview_status, debug_mode=False, return_json=True, use_playground=use_playground, current_project=current_project)
    return jsonify({'names': variables_json, 'vocabulary': list(vocab_list)})

@app.route('/api/stash_data', methods=['POST'])
@csrf.exempt
@cross_origin(origins='*', methods=['POST', 'HEAD'], automatic_options=True)
def api_stash_data():
    if not api_verify(request):
        return jsonify_with_status("Access denied.", 403)
    post_data = request.get_json(silent=True)
    if post_data is None:
        post_data = request.form.copy()
        if 'data' not in post_data:
            return jsonify_with_status("Data must be provided.", 400)
        try:
            data = json.loads(post_data['data'])
        except Exception as err:
            return jsonify_with_status("Malformed data.", 400)
    else:
        data = post_data['data']
    if not true_or_false(post_data.get('raw', False)):
        data = transform_json_variables(data)
    expire = post_data.get('expire', None)
    if expire is None:
        expire = 60*60*24*90
    try:
        expire = int(expire)
        assert expire > 0
    except:
        expire = 60*60*24*90
    (key, secret) = stash_data(data, expire)
    return jsonify({'stash_key': key, 'secret': secret})

@app.route('/api/retrieve_stashed_data', methods=['GET'])
@csrf.exempt
@cross_origin(origins='*', methods=['GET', 'HEAD'], automatic_options=True)
def api_retrieve_stashed_data():
    if not api_verify(request):
        return jsonify_with_status("Access denied.", 403)
    do_delete = true_or_false(request.args.get('delete', False))
    refresh = request.args.get('refresh', None)
    if refresh:
        try:
            refresh = int(refresh)
            assert refresh > 0
        except:
            refresh = False
    stash_key = request.args.get('stash_key', None)
    secret = request.args.get('secret', None)
    if stash_key is None or secret is None:
        return jsonify_with_status("The stash key and secret parameters are required.", 400)
    try:
        data = retrieve_stashed_data(stash_key, secret, delete=do_delete, refresh=refresh)
        assert data is not None
    except Exception as err:
        return jsonify_with_status("The stashed data could not be retrieved: " + err.__class__.__name__ + " " + str(err) + ".", 400)
    return jsonify(docassemble.base.functions.safe_json(data))

@app.route('/manage_api', methods=['GET', 'POST'])
@login_required
def manage_api():
    setup_translation()
    if not current_user.has_role(*daconfig.get('api privileges', ['admin', 'developer'])):
        return ('File not found', 404)
    form = APIKey(request.form)
    action = request.args.get('action', None)
    api_key = request.args.get('key', None)
    argu = {}
    argu['mode'] = 'list'
    if action is None:
        action = 'list'
    argu['form'] = form
    argu['extra_js'] = Markup("""
<script>
  function remove_constraint(elem){
    $(elem).parents('.daconstraintlist div').remove();
    fix_constraints();
  }
  function fix_constraints(){
    var empty;
    var filled_exist = 0;
    var empty_exist = 0;
    if ($("#method").val() == 'none'){
      $(".daconstraintlist").hide();
      return;
    }
    else{
      $(".daconstraintlist").show();
    }
    $(".daconstraintlist input").each(function(){
      if ($(this).val() == ''){
        empty_exist = 1;
      }
      else{
        filled_exist = 1;
      }
      if (!($(this).next().length)){
        var new_button = $('<button>');
        var new_i = $('<i>');
        $(new_button).addClass('btn btn-outline-secondary');
        $(new_i).addClass('fas fa-times');
        $(new_button).append(new_i);
        $(new_button).on('click', function(){remove_constraint(this);});
        $(this).parent().append(new_button);
      }
    });
    if (empty_exist == 0){
      var new_div = $('<div>');
      var new_input = $('<input>');
      $(new_div).append(new_input);
      $(new_div).addClass('input-group');
      $(new_input).addClass('form-control');
      $(new_input).attr('type', 'text');
      if ($("#method").val() == 'ip'){
        $(new_input).attr('placeholder', """ + json.dumps(word('e.g., 56.33.114.49')) + """);
      }
      else{
        $(new_input).attr('placeholder', """ + json.dumps(word('e.g., *example.com')) + """);
      }
      $(new_input).on('change', fix_constraints);
      $(new_input).on('keyup', fix_constraints);
      $(".daconstraintlist").append(new_div);
      var new_button = $('<button>');
      var new_i = $('<i>');
      $(new_button).addClass('btn btn-outline-secondary');
      $(new_i).addClass('fas fa-times');
      $(new_button).append(new_i);
      $(new_button).on('click', function(){remove_constraint(this);});
      $(new_div).append(new_button);
    }
  }
  $( document ).ready(function(){
    $(".daconstraintlist input").on('change', fix_constraints);
    $("#method").on('change', function(){
      $(".daconstraintlist div.input-group").remove();
      fix_constraints();
    });
    $("#submit").on('click', function(){
      var the_constraints = [];
      $(".daconstraintlist input").each(function(){
        if ($(this).val() != ''){
          the_constraints.push($(this).val());
        }
      });
      $("#security").val(JSON.stringify(the_constraints));
    });
    fix_constraints();
  });
</script>
""")
    form.method.choices = [('ip', 'IP Address'), ('referer', 'Referring URL'), ('none', 'No authentication')]
    ip_address = get_requester_ip(request)
    if request.method == 'POST' and form.validate():
        action = form.action.data
        try:
            constraints = json.loads(form.security.data)
            if not isinstance(constraints, list):
                constraints = []
        except:
            constraints = []
        if action == 'new':
            argu['title'] = word("New API Key")
            argu['tab_title'] = argu['title']
            argu['page_title'] = argu['title']
            info = dict(name=form.name.data, method=form.method.data, constraints=constraints)
            success = False
            for attempt in range(10):
                api_key = random_alphanumeric(32)
                info['last_four'] = api_key[-4:]
                new_api_key = encrypt_api_key(api_key)
                if not len(r.keys('da:apikey:userid:*:key:' + new_api_key + ':info')):
                    r.set('da:apikey:userid:' + str(current_user.id) + ':key:' + new_api_key + ':info', json.dumps(info))
                    success = True
                    break
            if not success:
                flash(word("Could not create new key"), 'error')
                return render_template('pages/manage_api.html', **argu)
            argu['description'] = Markup(word("Your new API key, known internally as %s, is %s") % (form.name.data, "<code>" + api_key + "</code>") + '.  ' + word("This is the only time you will be able to see your API key, so make sure to make a note of it and keep it in a secure place."))
        elif action == 'edit':
            argu['title'] = word("Edit API Key")
            argu['tab_title'] = argu['title']
            argu['page_title'] = argu['title']
            api_key = form.key.data
            argu['api_key'] = api_key
            rkey = 'da:apikey:userid:' + str(current_user.id) + ':key:' + str(form.key.data) + ':info'
            existing_key = r.get(rkey)
            if existing_key is None:
                flash(word("The key no longer exists"), 'error')
                return render_template('pages/manage_api.html', **argu)
            existing_key = existing_key.decode()
            if form.delete.data:
                r.delete(rkey)
                flash(word("The key was deleted"), 'info')
            else:
                try:
                    info = json.loads(existing_key)
                except:
                    flash(word("The key no longer exists"), 'error')
                    return render_template('pages/manage_api.html', **argu)
                info['name'] = form.name.data
                if form.method.data != info['method'] and form.method.data in ('ip', 'referer'):
                    info['method'] = form.method.data
                info['constraints'] = constraints
                r.set(rkey, json.dumps(info))
        action = 'list'
    if action == 'new':
        argu['title'] = word("New API Key")
        argu['tab_title'] = argu['title']
        argu['page_title'] = argu['title']
        argu['mode'] = 'new'
    if api_key is not None and action == 'edit':
        argu['title'] = word("Edit API Key")
        argu['tab_title'] = argu['title']
        argu['page_title'] = argu['title']
        argu['api_key'] = api_key
        argu['mode'] = 'edit'
        rkey = 'da:apikey:userid:' + str(current_user.id) + ':key:' + api_key + ':info'
        info = r.get(rkey)
        if info is not None:
            info = json.loads(info.decode())
            if isinstance(info, dict) and info.get('name', None) and info.get('method', None):
                argu['method'] = info.get('method')
                form.method.data = info.get('method')
                form.action.data = 'edit'
                form.key.data = api_key
                form.name.data = info.get('name')
                argu['constraints'] = info.get('constraints')
                argu['display_key'] = ('*' * 28) + info.get('last_four')
        if ip_address != '127.0.0.1':
            argu['description'] = Markup(word("Your IP address is") + " <code>" + str(ip_address) + "</code>.")
    if action == 'list':
        argu['title'] = word("API Keys")
        argu['tab_title'] = argu['title']
        argu['page_title'] = argu['title']
        argu['mode'] = 'list'
        avail_keys = []
        for rkey in r.keys('da:apikey:userid:' + str(current_user.id) + ':key:*:info'):
            rkey = rkey.decode()
            try:
                info = json.loads(r.get(rkey).decode())
                if not isinstance(info, dict):
                    logmessage("manage_api: response from redis was not a dict")
                    continue
            except:
                logmessage("manage_api: response from redis had invalid json")
                continue
            m = re.match(r'da:apikey:userid:[0-9]+:key:([^:]+):info', rkey)
            if not m:
                logmessage("manage_api: error with redis key")
                continue
            api_key = m.group(1)
            info['encoded_api_key'] = urllibquoteplus(api_key)
            avail_keys.append(info)
        argu['avail_keys'] = avail_keys
        argu['has_any_keys'] = bool(len(avail_keys) > 0)
    return render_template('pages/manage_api.html', **argu)

@app.route(html_index_path, methods=['GET'])
def html_index():
    resp = app.send_static_file('index.html')
    resp.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
    return resp

@app.route('/api/interview', methods=['GET', 'POST'])
@csrf.exempt
@cross_origin(origins='*', methods=['GET', 'POST', 'HEAD'], automatic_options=True)
def api_interview():
    if request.method == 'POST':
        post_data = request.get_json(silent=True)
        if post_data is None:
            return jsonify_with_status('The request must be JSON', 400)
        yaml_filename = post_data.get('i', None)
        secret = post_data.get('secret', None)
        session_id = post_data.get('session', None)
        url_args = post_data.get('url_args', None)
        user_code = post_data.get('user_code', None)
        command = post_data.get('command', None)
        referer = post_data.get('referer', None)
    else:
        yaml_filename = request.args.get('i', None)
        secret = request.args.get('secret', None)
        session_id = request.args.get('session', None)
        url_args = {}
        user_code = request.args.get('user_code', None)
        command = request.args.get('command', None)
        referer = request.args.get('referer', None)
        for key, val in request.args.items():
            if key not in ('session', 'secret', 'i', 'user_code', 'command', 'referer', 'action'):
                url_args[key] = val
        if len(url_args) == 0:
            url_args = None
    output = {}
    action = None
    reset_fully = False
    is_new = False
    changed = False
    if user_code:
        key = 'da:apiinterview:usercode:' + user_code
        user_info = r.get(key)
        if user_info is None:
            user_code = None
        else:
            r.expire(key, 60*60*24*30)
            try:
                user_info = json.loads(user_info)
            except:
                user_code = None
        if user_code:
            if user_info['user_id']:
                user = db.session.execute(select(UserModel).filter_by(id=user_info['user_id'])).scalar()
                if user is None or user.social_id.startswith('disabled$') or not user.active:
                    user_code = None
                else:
                    login_user(user, remember=False)
                    update_last_login(user)
            else:
                session['tempuser'] = user_info['temp_user_id']
    if not user_code:
        user_code = app.session_interface.manual_save_session(app, session).decode()
        if current_user.is_anonymous:
            new_temp_user = TempUser()
            db.session.add(new_temp_user)
            db.session.commit()
            session['tempuser'] = new_temp_user.id
            user_info = {"user_id": None, "temp_user_id": new_temp_user.id, "sessions": {}}
        else:
            user_info = {"user_id": current_user.id, "temp_user_id": None, "sessions": {}}
        output['user_code'] = user_code
        changed = True
    need_to_reset = False
    new_session = False
    send_initial = False
    if yaml_filename.startswith('/'):
        parts = urlparse(yaml_filename)
        params = urllib.parse.parse_qs(parts.query)
        if params.get('action', '') != '':
            try:
                action = tidy_action(json.loads(myb64unquote(params['action'])))
            except:
                return jsonify_with_status(word("Invalid action."), 400)
        url_args = {}
        for key, val in dict(params).items():
            params[key] = val[0]
            if key not in reserved_argnames:
                url_args[key] = val[0]
        if parts.path == '/launch':
            code = params.get('c', None)
            if code is None:
                abort(403)
            the_key = 'da:resume_interview:' + str(code)
            data = r.get(the_key)
            if data is None:
                return jsonify_with_status(word("The link has expired."), 403)
            data = json.loads(data.decode())
            if data.get('once', False):
                r.delete(the_key)
            args = {}
            for key, val in params.items():
                if key != 'session':
                    args[key] = val
            yaml_filename = data['i']
            if 'session' in data:
                session_id = data['session']
                user_info['sessions'][yaml_filename] = session_id
            else:
                new_session = True
        if parts.path in ('/i', '/interview', '/'):
            ok = False
            if 'i' in params:
                yaml_filename = params['i']
                ok = True
            elif 'state' in params:
                try:
                    yaml_filename = re.sub(r'\^.*', '', from_safeid(params['state']))
                    ok = True
                except:
                    ok = False
            if not ok:
                if current_user.is_anonymous and not daconfig.get('allow anonymous access', True):
                    output['redirect'] = url_for('user.login')
                    return jsonify(output)
                if len(daconfig['dispatch']) > 0:
                    output['redirect'] = url_for('interview_start')
                    return jsonify(output)
                else:
                    yaml_filename = final_default_yaml_filename
        refer = None
        if parts.path.startswith('/start/') or parts.path.startswith('/run/'):
            m = re.search(r'/(start|run)/([^/]+)/$', parts.path)
            if m:
                refer = [m.group(1) + '_dispatch', m.group(2)]
                dispatch = m.group(2)
            else:
                m = re.search(r'/(start|run)/([^/]+)/([^/]+)/(.*)/$', parts.path)
                if m:
                    refer = [m.group(1) + '_directory', m.group(2), m.group(3), m.group(4)]
                    yaml_filename = 'docassemble.' + m.group(2) + ':data/questions/' + m.group(3) + '/' + m.group(4) + '.yml'
                else:
                    m = re.search(r'/(start|run)/([^/]+)/(.*)/$', parts.path)
                    if m:
                        refer = [m.group(1), m.group(2), m.group(3)]
                        if re.search(r'playground[0-9]', m.group(2)):
                            yaml_filename = 'docassemble.' + m.group(2) + ':' + m.group(3) + '.yml'
                        else:
                            yaml_filename = 'docassemble.' + m.group(2) + ':data/questions/' + m.group(3) + '.yml'
                    else:
                        yaml_filename = None
            if yaml_filename is None:
                return jsonify_with_status("File not found", 404)
            if m.group(1) == 'start':
                new_session = True
        if true_or_false(params.get('reset', False)):
            need_to_reset = True
            if str(params['reset']) == '2':
                reset_fully = True
        if true_or_false(params.get('new_session', False)):
            new_session = True
        index_params = dict(i=yaml_filename)
        output['i'] = yaml_filename
        output['page_sep'] = "#page"
        if refer is None:
            output['location_bar'] = url_for('index', **index_params)
        elif refer[0] in ('start', 'run'):
            output['location_bar'] = url_for('run_interview_in_package', package=refer[1], filename=refer[2])
            output['page_sep'] = "#/"
        elif refer[0] in ('start_dispatch', 'run_dispatch'):
            output['location_bar'] = url_for('run_interview', dispatch=refer[1])
            output['page_sep'] = "#/"
        elif refer[0] in ('start_directory', 'run_directory'):
            output['location_bar'] = url_for('run_interview_in_package_directory', package=refer[1], directory=refer[2], filename=refer[3])
            output['page_sep'] = "#/"
        else:
            output['location_bar'] = None
            for k, v in daconfig['dispatch'].items():
                if v == yaml_filename:
                    output['location_bar'] = url_for('run_interview', dispatch=k)
                    output['page_sep'] = "#/"
                    break
            if output['location_bar'] is None:
                output['location_bar'] = url_for('index', **index_params)
        send_initial = True
    if not yaml_filename:
        return jsonify_with_status("Parameter i is required.", 400)
    if not secret:
        secret = random_string(16)
        output['secret'] = secret
    secret = str(secret)
    docassemble.base.functions.this_thread.current_info = current_info(req=request, interface='api', secret=secret)
    if yaml_filename not in user_info['sessions'] or need_to_reset or new_session:
        was_new = True
        if PREVENT_DEMO and (yaml_filename.startswith('docassemble.base:') or yaml_filename.startswith('docassemble.demo:')) and (current_user.is_anonymous or not current_user.has_role('admin', 'developer')):
            return jsonify_with_status(word("Not authorized"), 403)
        if current_user.is_anonymous and not daconfig.get('allow anonymous access', True):
            output['redirect'] = url_for('user.login', next=url_for('index', i=yaml_filename, **url_args))
            return jsonify(output)
        if yaml_filename.startswith('docassemble.playground'):
            if not app.config['ENABLE_PLAYGROUND']:
                return jsonify_with_status(word("Not authorized"), 403)
        else:
            yaml_filename = re.sub(r':([^\/]+)$', r':data/questions/\1', yaml_filename)
        interview = docassemble.base.interview_cache.get_interview(yaml_filename)
        if session_id is None:
            if need_to_reset and yaml_filename in user_info['sessions']:
                reset_user_dict(user_info['sessions'][yaml_filename], yaml_filename)
                del user_info['sessions'][yaml_filename]
            unique_sessions = interview.consolidated_metadata.get('sessions are unique', False)
            if unique_sessions is not False and not current_user.is_authenticated:
                if yaml_filename in user_info['sessions']:
                    del user_info['sessions'][yaml_filename]
                output['redirect'] = url_for('user.login', next=url_for('index', i=yaml_filename, **url_args))
                return jsonify(output)
            if interview.consolidated_metadata.get('temporary session', False):
                if yaml_filename in user_info['sessions']:
                    reset_user_dict(user_info['sessions'][yaml_filename], yaml_filename)
                    del user_info['sessions'][yaml_filename]
                if current_user.is_authenticated:
                    while True:
                        the_session_id, encrypted = get_existing_session(yaml_filename, secret)
                        if the_session_id:
                            reset_user_dict(the_session_id, yaml_filename)
                        else:
                            break
                    need_to_reset = True
            if current_user.is_anonymous:
                if (not interview.allowed_to_initiate(is_anonymous=True)) or (not interview.allowed_to_access(is_anonymous=True)):
                    output['redirect'] = url_for('user.login', next=url_for('index', i=yaml_filename, **url_args))
                    return jsonify(output)
            elif not interview.allowed_to_initiate(has_roles=[role.name for role in current_user.roles]):
                return jsonify_with_status(word("You are not allowed to access this interview."), 403)
            elif not interview.allowed_to_access(has_roles=[role.name for role in current_user.roles]):
                return jsonify_with_status(word("You are not allowed to access this interview."), 403)
            session_id = None
            if reset_fully:
                user_info['sessions'] = {}
            if (not need_to_reset) and (unique_sessions is True or (isinstance(unique_sessions, list) and len(unique_sessions) and current_user.has_role(*unique_sessions))):
                session_id, encrypted = get_existing_session(yaml_filename, secret)
        else:
            unique_sessions = interview.consolidated_metadata.get('sessions are unique', False)
            if unique_sessions is not False and not current_user.is_authenticated:
                if yaml_filename in user_info['sessions']:
                    del user_info['sessions'][yaml_filename]
                output['redirect'] = url_for('user.login', next=url_for('index', i=yaml_filename, session=session_id, **url_args))
                return jsonify(output)
            if current_user.is_anonymous:
                if (not interview.allowed_to_initiate(is_anonymous=True)) or (not interview.allowed_to_access(is_anonymous=True)):
                    output['redirect'] = url_for('user.login', next=url_for('index', i=yaml_filename, session=session_id, **url_args))
                    return jsonify(output)
            elif not interview.allowed_to_initiate(has_roles=[role.name for role in current_user.roles]):
                if yaml_filename in user_info['sessions']:
                    del user_info['sessions'][yaml_filename]
                return jsonify_with_status(word("You are not allowed to access this interview."), 403)
            elif not interview.allowed_to_access(has_roles=[role.name for role in current_user.roles]):
                if yaml_filename in user_info['sessions']:
                    del user_info['sessions'][yaml_filename]
                return jsonify_with_status(word("You are not allowed to access this interview."), 403)
            if need_to_reset:
                reset_user_dict(session_id, yaml_filename)
        session_id = None
    if new_session:
        session_id = None
        if yaml_filename in user_info['sessions']:
            del user_info['sessions'][yaml_filename]
    if not session_id:
        if yaml_filename in user_info['sessions']:
            session_id = user_info['sessions'][yaml_filename]
        else:
            try:
                (encrypted, session_id) = create_new_interview(yaml_filename, secret, url_args=url_args, referer=referer, req=request)
            except Exception as err:
                return jsonify_with_status(err.__class__.__name__ + ': ' + str(err), 400)
            user_info['sessions'][yaml_filename] = session_id
            changed = True
            is_new = True
        #output['session'] = session_id
    if changed:
        key = 'da:apiinterview:usercode:' + user_code
        pipe = r.pipeline()
        pipe.set(key, json.dumps(user_info))
        pipe.expire(key, 60*60*24*30)
        pipe.execute()
    if not is_new:
        if url_args is not None and isinstance(url_args, dict) and len(url_args) > 0:
            logmessage("url_args is " + repr(url_args))
            variables = {}
            for key, val in url_args.items():
                variables["url_args[%s]" % (repr(key),)] = val
            try:
                set_session_variables(yaml_filename, session_id, variables, secret=secret, use_lock=True)
            except Exception as the_err:
                return jsonify_with_status(str(the_err), 400)
    obtain_lock(session_id, yaml_filename)
    if request.method == 'POST' and command == 'action':
        action = post_data.get('action', None)
    if action is not None:
        if not isinstance(action, dict) or 'action' not in action or 'arguments' not in action:
            release_lock(session_id, yaml_filename)
            return jsonify_with_status("Invalid action", 400)
        try:
            data = get_question_data(yaml_filename, session_id, secret, save=True, use_lock=False, action=action, post_setting=True, advance_progress_meter=True, encode=True)
        except Exception as err:
            release_lock(session_id, yaml_filename)
            return jsonify_with_status(str(err), 400)
    else:
        try:
            data = get_question_data(yaml_filename, session_id, secret, save=False, use_lock=False, encode=True)
        except Exception as err:
            release_lock(session_id, yaml_filename)
            return jsonify_with_status(str(err), 400)
    if request.method == 'POST':
        if command == 'back':
            if data['allow_going_back']:
                try:
                    data = go_back_in_session(yaml_filename, session_id, secret=secret, return_question=True, encode=True)
                except Exception as the_err:
                    release_lock(session_id, yaml_filename)
                    return jsonify_with_status(str(the_err), 400)
        elif command is None:
            variables = post_data.get('variables', None)
            if not isinstance(variables, dict):
                release_lock(session_id, yaml_filename)
                return jsonify_with_status("variables must be a dictionary", 400)
            if variables is not None:
                variables = transform_json_variables(variables)
            valid_variables = {}
            if 'fields' in data:
                for field in data['fields']:
                    if 'variable_name' in field and field.get('active', False):
                        valid_variables[field['variable_name']] = field
                    if field.get('required', False) and 'variable_name' in field:
                        if field['variable_name'] not in variables:
                            release_lock(session_id, yaml_filename)
                            return jsonify_with_status("variable %s is missing" % (field['variable_name'],), 400)
            for key, val in variables.items():
                if key not in valid_variables:
                    release_lock(session_id, yaml_filename)
                    return jsonify_with_status("invalid variable name " + repr(key), 400)
            try:
                data = set_session_variables(yaml_filename, session_id, variables, secret=secret, return_question=True, event_list=data.get('event_list', None), question_name=data.get('questionName', None), encode=True)
            except Exception as the_err:
                release_lock(session_id, yaml_filename)
                return jsonify_with_status(str(the_err), 400)
        elif command != 'action':
            release_lock(session_id, yaml_filename)
            return jsonify_with_status("Invalid command", 400)
    if data.get('questionType', None) in ('response', 'sendfile'):
        output['question'] = {
            'questionType': data['questionType']
        }
    else:
        output['question'] = data
    release_lock(session_id, yaml_filename)
    if send_initial:
        output['setup'] = {}
        if 'google maps api key' in google_config:
            api_key = google_config.get('google maps api key')
        elif 'api key' in google_config:
            api_key = google_config.get('api key')
        else:
            api_key = None
        if api_key:
            output['setup']['googleApiKey'] = api_key
        if ga_configured and data['interview_options'].get('analytics on', True):
            interview_package = re.sub(r'^docassemble\.', '', re.sub(r':.*', '', yaml_filename))
            interview_filename = re.sub(r'\.ya?ml$', '', re.sub(r'.*[:\/]', '', yaml_filename), re.IGNORECASE)
            output['setup']['googleAnalytics'] = dict(enable=True, ga_id=google_config.get('analytics id'), prefix=interview_package + '/' + interview_filename)
        else:
            output['setup']['googleAnalytics'] = dict(enable=False)
      #<script src="https://maps.googleapis.com/maps/api/js?key=' + api_key + '&libraries=places"></script>'
      #window.dataLayer = window.dataLayer || [];
      #function gtag(){dataLayer.push(arguments);}
      #gtag('js', new Date());
      #function daPageview(){
      #  var idToUse = daQuestionID['id'];
      #  if (daQuestionID['ga'] != undefined && daQuestionID['ga'] != null){
      #    idToUse = daQuestionID['ga'];
      #  }
      #  if (idToUse != null){
      #    gtag('config', """ + json.dumps(ga_id) + """, {""" + ("'cookie_flags': 'SameSite=None;Secure', " if app.config['SESSION_COOKIE_SECURE'] else '') + """'page_path': """ + json.dumps(interview_package) + """ + "/" + """ + json.dumps(interview_filename) + """ + "/" + idToUse.replace(/[^A-Za-z0-9]+/g, '_')});
      #  }
      #}
      #<script async src="https://www.googletagmanager.com/gtag/js?id=""" + ga_id + """"></script>
    return jsonify(output)

@app.route('/me', methods=['GET'])
def whoami():
    if current_user.is_authenticated:
        return jsonify(logged_in=True, user_id=current_user.id, email=current_user.email, roles=[role.name for role in current_user.roles], firstname=current_user.first_name, lastname=current_user.last_name, country=current_user.country, subdivisionfirst=current_user.subdivisionfirst, subdivisionsecond=current_user.subdivisionsecond, subdivisionthird=current_user.subdivisionthird, organization=current_user.organization, timezone=current_user.timezone)
    else:
        return jsonify(logged_in=False)

def retrieve_email(email_id):
    if not isinstance(email_id, int):
        raise DAError("email_id not provided")
    email = db.session.execute(select(Email).filter_by(id=email_id)).scalar()
    if email is None:
        raise DAError("E-mail did not exist")
    short_record = db.session.execute(select(Shortener).filter_by(short=email.short)).scalar()
    if short_record is not None and short_record.user_id is not None:
        user = db.session.execute(select(UserModel).options(db.joinedload(UserModel.roles)).filter_by(id=short_record.user_id, active=True)).scalar()
    else:
        user = None
    if short_record is None:
        raise DAError("Short code did not exist")
    return get_email_obj(email, short_record, user)

class AddressEmail:
    def __str__(self):
        return str(self.address)

def retrieve_emails(**pargs):
    key = pargs.get('key', None)
    index = pargs.get('index', None)
    if key is None and index is not None:
        raise DAError("retrieve_emails: if you provide an index you must provide a key")
    if 'i' in pargs:
        yaml_filename = pargs['i']
    else:
        yaml_filename = docassemble.base.functions.this_thread.current_info.get('yaml_filename', None)
    if 'uid' in pargs:
        uid = pargs['uid']
    else:
        uid = docassemble.base.functions.get_uid()
    if 'user_id' in pargs:
        user_id = pargs['user_id']
        temp_user_id = None
    elif 'temp_user_id' in pargs:
        user_id = None
        temp_user_id = pargs['temp_user_id']
    elif current_user.is_anonymous:
        user_id = None
        temp_user_id = session.get('tempuser', None)
    else:
        user_id = current_user.id
        temp_user_id = None
    user_cache = {}
    results = []
    if key is None:
        the_query = db.session.execute(select(Shortener).filter_by(filename=yaml_filename, uid=uid, user_id=user_id, temp_user_id=temp_user_id).order_by(Shortener.modtime)).scalars()
    else:
        if index is None:
            the_query = db.session.execute(select(Shortener).filter_by(filename=yaml_filename, uid=uid, user_id=user_id, temp_user_id=temp_user_id, key=key).order_by(Shortener.modtime)).scalars()
        else:
            the_query = db.session.execute(select(Shortener).filter_by(filename=yaml_filename, uid=uid, user_id=user_id, temp_user_id=temp_user_id, key=key, index=index).order_by(Shortener.modtime)).scalars()
    for record in the_query:
        result_for_short = AddressEmail()
        result_for_short.address = record.short
        result_for_short.key = record.key
        result_for_short.index = record.index
        result_for_short.emails = []
        if record.user_id is not None:
            if record.user_id in user_cache:
                user = user_cache[record.user_id]
            else:
                user = get_user_object(record.user_id)
                user_cache[record.user_id] = user
            result_for_short.owner = user.email
        else:
            user = None
            result_for_short.owner = None
        for email in db.session.execute(select(Email).filter_by(short=record.short).order_by(Email.datetime_received)).scalars():
            result_for_short.emails.append(get_email_obj(email, record, user))
        results.append(result_for_short)
    return results

def get_email_obj(email, short_record, user):
    email_obj = DAEmail(short=email.short)
    email_obj.key = short_record.key
    email_obj.index = short_record.index
    email_obj.initializeAttribute('to_address', DAEmailRecipientList, json.loads(email.to_addr), gathered=True)
    email_obj.initializeAttribute('cc_address', DAEmailRecipientList, json.loads(email.cc_addr), gathered=True)
    email_obj.initializeAttribute('from_address', DAEmailRecipient, **json.loads(email.from_addr))
    email_obj.initializeAttribute('reply_to', DAEmailRecipient, **json.loads(email.reply_to_addr))
    email_obj.initializeAttribute('return_path', DAEmailRecipient, **json.loads(email.return_path_addr))
    email_obj.subject = email.subject
    email_obj.datetime_message = email.datetime_message
    email_obj.datetime_received = email.datetime_received
    email_obj.initializeAttribute('attachment', DAFileList, gathered=True)
    if user is None:
        email_obj.address_owner = None
    else:
        email_obj.address_owner = user.email
    for attachment_record in db.session.execute(select(EmailAttachment).filter_by(email_id=email.id).order_by(EmailAttachment.index)).scalars():
        #sys.stderr.write("Attachment record is " + str(attachment_record.id) + "\n")
        upload = db.session.execute(select(Uploads).filter_by(indexno=attachment_record.upload)).scalar()
        if upload is None:
            continue
        #sys.stderr.write("Filename is " + upload.filename + "\n")
        saved_file_att = SavedFile(attachment_record.upload, extension=attachment_record.extension, fix=True)
        process_file(saved_file_att, saved_file_att.path, attachment_record.content_type, attachment_record.extension, initial=False)
        extension, mimetype = get_ext_and_mimetype(upload.filename)
        if upload.filename == 'headers.json':
            #sys.stderr.write("Processing headers\n")
            email_obj.initializeAttribute('headers', DAFile, mimetype=mimetype, extension=extension, number=attachment_record.upload)
        elif upload.filename == 'attachment.txt' and attachment_record.index < 3:
            #sys.stderr.write("Processing body text\n")
            email_obj.initializeAttribute('body_text', DAFile, mimetype=mimetype, extension=extension, number=attachment_record.upload)
        elif upload.filename == 'attachment.html' and attachment_record.index < 3:
            email_obj.initializeAttribute('body_html', DAFile, mimetype=mimetype, extension=extension, number=attachment_record.upload)
        else:
            email_obj.attachment.appendObject(DAFile, mimetype=mimetype, extension=extension, number=attachment_record.upload)
    if not hasattr(email_obj, 'headers'):
        email_obj.headers = None
    if not hasattr(email_obj, 'body_text'):
        email_obj.body_text = None
    if not hasattr(email_obj, 'body_html'):
        email_obj.body_html = None
    return email_obj

def da_send_fax(fax_number, the_file, config, country=None):
    if clicksend_config is not None:
        if config not in clicksend_config['name']:
            raise Exception("There is no ClickSend configuration called " + str(config))
        info = docassemble.webapp.clicksend.send_fax(fax_number, the_file, clicksend_config['name'][config], country)
        the_key = 'da:faxcallback:sid:' + info['message_id']
        pipe = r.pipeline()
        pipe.set(the_key, json.dumps(info))
        pipe.expire(the_key, 86400)
        pipe.execute()
        return info['message_id']
    if twilio_config is None:
        logmessage("da_send_fax: ignoring call to da_send_fax because Twilio not enabled")
        return None
    if config not in twilio_config['name'] or 'fax' not in twilio_config['name'][config] or twilio_config['name'][config]['fax'] in (False, None):
        logmessage("da_send_fax: ignoring call to da_send_fax because fax feature not enabled")
        return None
    account_sid = twilio_config['name'][config].get('account sid', None)
    auth_token = twilio_config['name'][config].get('auth token', None)
    from_number = twilio_config['name'][config].get('number', None)
    if account_sid is None or auth_token is None or from_number is None:
        logmessage("da_send_fax: ignoring call to da_send_fax because account sid, auth token, and/or number missing")
        return None
    client = TwilioRestClient(account_sid, auth_token)
    fax = client.fax.v1.faxes.create(
        from_=from_number,
        to=fax_number,
        media_url=the_file.url_for(temporary=True, seconds=600),
        status_callback=url_for('fax_callback', _external=True)
    )
    return fax.sid

def write_pypirc():
    pypirc_file = daconfig.get('pypirc path', '/var/www/.pypirc')
    #pypi_username = daconfig.get('pypi username', None)
    #pypi_password = daconfig.get('pypi password', None)
    pypi_url = daconfig.get('pypi url', 'https://upload.pypi.org/legacy/')
    # if pypi_username is None or pypi_password is None:
    #     return
    if os.path.isfile(pypirc_file):
        with open(pypirc_file, 'r', encoding='utf-8') as fp:
            existing_content = fp.read()
    else:
        existing_content = None
    content = """\
[distutils]
index-servers =
  pypi

[pypi]
repository: """ + pypi_url + "\n"
#     """
# username: """ + pypi_username + """
# password: """ + pypi_password + "\n"
    if existing_content != content:
        with open(pypirc_file, 'w', encoding='utf-8') as fp:
            fp.write(content)
        os.chmod(pypirc_file, stat.S_IRUSR | stat.S_IWUSR)

def url_sanitize(url):
    return re.sub(r'\s', ' ', url)

def pypi_status(packagename, limitation=None):
    result = {}
    pypi_url = daconfig.get('pypi url', 'https://pypi.python.org/pypi')
    try:
        response = requests.get(url_sanitize(pypi_url + '/' + str(packagename) + '/json'))
        assert response.status_code == 200
    except AssertionError:
        if response.status_code == 404:
            result['error'] = False
            result['exists'] = False
        else:
            result['error'] = response.status_code
    except:
        result['error'] = 'unknown'
    else:
        try:
            result['info'] = response.json()
        except:
            result['error'] = 'json'
        else:
            result['error'] = False
            result['exists'] = True
    return result

def page_after_login():
    if current_user.is_authenticated:
        for role, page in daconfig['page after login']:
            if role == '*' or current_user.has_role(role):
                return page
    return 'interview_list'

def path_from_reference(file_reference):
    if isinstance(file_reference, DAFileCollection):
        file_reference = file_reference._first_file()
    if isinstance(file_reference, DAFileList):
        file_reference = file_reference[0]
    if isinstance(file_reference, DAFile):
        file_info = get_info_from_file_number_with_uids(file_reference.number)
        if 'fullpath' not in file_info:
            raise Exception("File not found")
        path = file_info['fullpath']
        friendly_path = os.path.join(tempfile.mkdtemp(prefix='SavedFile'), file_reference.filename)
        try:
            os.symlink(file_info['fullpath'], friendly_path)
        except:
            shutil.copyfile(file_info['fullpath'], friendly_path)
        return friendly_path
    if isinstance(file_reference, DAStaticFile):
        return file_reference.path()
    if file_reference is None:
        return None
    file_info = get_info_from_file_reference(file_reference)
    if 'fullpath' not in file_info:
        raise Exception("File not found")
    return file_info['fullpath']

def secure_filename_spaces_ok(filename):
    filename = unicodedata.normalize("NFKD", filename)
    filename = filename.encode("ascii", "ignore").decode("ascii")
    for sep in os.path.sep, os.path.altsep:
        if sep:
            filename = filename.replace(sep, "_")
    filename = str(re.sub(r'[^A-Za-z0-9\_\.\- ]', '', " ".join(filename.split(' ')))).strip("._ ")
    return filename

def secure_filename(filename):
    filename = werkzeug.utils.secure_filename(filename)
    extension, mimetype = get_ext_and_mimetype(filename)
    filename = re.sub(r'\.[^\.]+$', '', filename) + '.' + extension
    return filename

def sanitize_arguments(*pargs):
    for item in pargs:
        if isinstance(item, str):
            if item.startswith('/') or item.startswith('.') or re.search(r'\s', item):
                raise Exception("Invalid parameter " + item)

def get_short_code(**pargs):
    key = pargs.get('key', None)
    index = pargs.get('index', None)
    if key is None and index is not None:
        raise DAError("get_short_code: if you provide an index you must provide a key")
    if 'i' in pargs:
        yaml_filename = pargs['i']
    else:
        yaml_filename = docassemble.base.functions.this_thread.current_info.get('yaml_filename', None)
    if 'uid' in pargs:
        uid = pargs['uid']
    else:
        uid = docassemble.base.functions.get_uid()
    if 'user_id' in pargs:
        user_id = pargs['user_id']
        temp_user_id = None
    elif 'temp_user_id' in pargs:
        user_id = None
        temp_user_id = pargs['temp_user_id']
    elif current_user.is_anonymous:
        user_id = None
        temp_user_id = session.get('tempuser', None)
    else:
        user_id = current_user.id
        temp_user_id = None
    short_code = None
    for record in db.session.execute(select(Shortener.short).filter_by(filename=yaml_filename, uid=uid, user_id=user_id, temp_user_id=temp_user_id, key=key, index=index)):
        short_code = record.short
    if short_code is not None:
        return short_code
    counter = 0
    new_record = None
    while counter < 20:
        existing_id = None
        new_short = random_lower_string(6)
        for record in db.session.execute(select(Shortener.id).filter_by(short=new_short)):
            existing_id = record.id
        if existing_id is None:
            new_record = Shortener(filename=yaml_filename, uid=uid, user_id=user_id, temp_user_id=temp_user_id, short=new_short, key=key, index=index)
            db.session.add(new_record)
            db.session.commit()
            break
        counter += 1
    if new_record is None:
        raise SystemError("Failed to generate unique short code")
    return new_short

def illegal_variable_name(var):
    if re.search(r'[\n\r]', var):
        return True
    try:
        t = ast.parse(var)
    except:
        return True
    detector = docassemble.base.astparser.detectIllegal()
    detector.visit(t)
    return detector.illegal

emoji_match = re.compile(r':([A-Za-z][A-Za-z0-9\_\-]+):')
html_match = re.compile(r'(</?[A-Za-z\!][^>]*>|https*://[A-Za-z0-9\-\_:\%\/\@\.\#\&\=\~\?]+|mailto*://[A-Za-z0-9\-\_:\%\/\@\.\#\&\=\~]+\?)')

def mako_parts(expression):
    in_percent = False
    in_var = False
    in_square = False
    var_depth = 0
    in_colon = 0
    in_html = 0
    in_pre_bracket = False
    in_post_bracket = False
    output = []
    current = ''
    i = 0
    expression = emoji_match.sub(r'^^\1^^', expression)
    expression = html_match.sub(r'!@\1!@', expression)
    n = len(expression)
    while i < n:
        if in_html:
            if i + 1 < n and expression[i:i+2] == '!@':
                in_html = False
                if current != '':
                    output.append([current, 2])
                current = ''
                i += 2
            else:
                current += expression[i]
                i += 1
            continue
        if in_percent:
            if expression[i] in ["\n", "\r"]:
                in_percent = False
                current += expression[i]
                output.append([current, 1])
                current = ''
                i += 1
                continue
        elif in_var:
            if expression[i] == '{' and expression[i-1] != "\\":
                var_depth += 1
            elif expression[i] == '}' and expression[i-1] != "\\":
                var_depth -= 1
                if var_depth == 0:
                    current += expression[i]
                    if current != '':
                        output.append([current, 2])
                    current = ''
                    in_var = False
                    i += 1
                    continue
        elif in_pre_bracket:
            if i + 2 < n:
                if expression[i:i+3] == '</%':
                    in_pre_bracket = False
                    in_post_bracket = True
                    current += expression[i:i+3]
                    i += 3
                    continue
            if i + 1 < n and expression[i:i+2] == '%>':
                in_pre_bracket = False
                current += expression[i:i+2]
                if current != '':
                    output.append([current, 1])
                current = ''
                i += 2
                continue
        elif in_post_bracket:
            if expression[i] == '>' and expression[i-1] != "\\":
                current += expression[i]
                if current != '':
                    output.append([current, 1])
                current = ''
                in_post_bracket = False
                i += 1
                continue
        elif in_square:
            if expression[i] == ']' and (i == 0 or expression[i-1] != "\\"):
                mode = 0
                current += expression[i]
                for pattern in ['[FILE', '[TARGET ', '[EMOJI ', '[QR ', '[YOUTUBE', '[VIMEO]', '[PAGENUM]', '[BEGIN_TWOCOL]', '[BREAK]', '[END_TWOCOL', '[BEGIN_CAPTION]', '[VERTICAL_LINE]', '[END_CAPTION]', '[TIGHTSPACING]', '[SINGLESPACING]', '[DOUBLESPACING]', '[ONEANDAHALFSPACING]', '[TRIPLESPACING]', '[START_INDENTATION]', '[STOP_INDENTATION]', '[NBSP]', '[REDACTION', '[ENDASH]', '[EMDASH]', '[HYPHEN]', '[CHECKBOX]', '[BLANK]', '[BLANKFILL]', '[PAGEBREAK]', '[PAGENUM]', '[SECTIONNUM]', '[SKIPLINE]', '[NEWLINE]', '[NEWPAR]', '[BR]', '[TAB]', '[END]', '[BORDER]', '[NOINDENT]', '[FLUSHLEFT]', '[FLUSHRIGHT]', '[CENTER]', '[BOLDCENTER]', '[INDENTBY', '[${']:
                    if current.startswith(pattern):
                        mode = 2
                        break
                if current != '':
                    output.append([current, mode])
                current = ''
                in_square = False
                i += 1
                continue
            if i + 1 < n and expression[i:i+2] == '^^':
                if in_colon:
                    in_colon = False
                    current += ':'
                    output.append([current, 2])
                    current = ''
                else:
                    in_colon = True
                    if current.startswith('[${'):
                        output.append([current, 2])
                    else:
                        output.append([current, 0])
                    current = ':'
                i += 2
                continue
            if i + 1 < n and expression[i:i+2] == '!@':
                in_html = True
                if current != '':
                    if current.startswith('[${'):
                        output.append([current, 2])
                    else:
                        output.append([current, 0])
                current = ''
                i += 2
                continue
        elif in_colon:
            if i + 1 < n and expression[i:i+2] == '^^':
                current += ':'
                if current != '':
                    output.append([current, 2])
                current = ''
                in_colon = False
                i += 2
                continue
        elif i + 1 < n:
            if expression[i:i+2] == '${':
                in_var = True
                var_depth += 1
                if current != '':
                    output.append([current, 0])
                current = expression[i:i+2]
                i += 2
                continue
            if expression[i:i+2] == '^^':
                in_colon = True
                if current != '':
                    output.append([current, 0])
                current = ':'
                i += 2
                continue
            if expression[i:i+2] == '!@':
                in_html = True
                if current != '':
                    output.append([current, 0])
                current = ''
                i += 2
                continue
            if expression[i:i+2] == '<%':
                in_pre_bracket = True
                if current != '':
                    output.append([current, 0])
                current = expression[i:i+2]
                i += 2
                continue
            if expression[i:i+2] == '% ' and start_of_line(expression, i):
                in_percent = True
                if current != '':
                    output.append([current, 0])
                current = expression[i:i+2]
                i += 2
                continue
            if expression[i] == '[' and (i == 0 or expression[i-1] != "\\"):
                in_square = True
                if current != '':
                    output.append([current, 0])
                current = expression[i]
                i += 1
                continue
        current += expression[i]
        i += 1
    if current != '':
        if in_pre_bracket or in_post_bracket or in_percent:
            output.append([current, 1])
        elif in_var:
            output.append([current, 2])
        else:
            output.append([current, 0])
    return output

def start_of_line(expression, i):
    if i == 0:
        return True
    i -= 1
    while i >= 0:
        if expression[i] in ("\n", "\r"):
            return True
        if expression[i] in (" ", "\t"):
            i -= 1
            continue
        return False
    return True

def applock(action, application):
    key = 'da:applock:' + application + ':' + hostname
    if action == 'obtain':
        found = False
        count = 4
        while count > 0:
            record = r.get(key)
            if record:
                sys.stderr.write("obtain_applock: waiting for " + key + "\n")
                time.sleep(1.0)
            else:
                found = False
                break
            found = True
            count -= 1
        if found:
            sys.stderr.write("Request for applock " + key + " deadlocked\n")
            r.delete(key)
        pipe = r.pipeline()
        pipe.set(key, 1)
        pipe.expire(key, 4)
        pipe.execute()
    elif action == 'release':
        r.delete(key)

@app.errorhandler(CSRFError)
def handle_csrf_error(the_error):
    if request.method == 'POST':
        setup_translation()
        if 'ajax' in request.form and int(request.form['ajax']):
            flash(word("Input not processed because the page expired."), "success")
            return jsonify(dict(action='reload', reason='csrf_error'))
        try:
            referer = str(request.referrer)
        except:
            referer = None
        if referer:
            flash(word("Input not processed because the page expired."), "success")
            return redirect(referer)
    return server_error(the_error)

def error_notification(err, message=None, history=None, trace=None, referer=None, the_request=None, the_vars=None):
    recipient_email = daconfig.get('error notification email', None)
    if not recipient_email:
        return
    if err.__class__.__name__ in ['CSRFError', 'ClientDisconnected', 'MethodNotAllowed', 'DANotFoundError'] + ERROR_TYPES_NO_EMAIL:
        return
    email_recipients = []
    if isinstance(recipient_email, list):
        email_recipients.extend(recipient_email)
    else:
        email_recipients.append(recipient_email)
    if message is None:
        errmess = str(err)
    else:
        errmess = message
    try:
        email_address = current_user.email
    except:
        email_address = None
    if the_request:
        try:
            referer = str(the_request.referrer)
        except:
            referer = None
        ipaddress = get_requester_ip(the_request)
    else:
        referer = None
        ipaddress = None
    if daconfig.get('error notification variables', DEBUG):
        if the_vars is None:
            try:
                the_vars = docassemble.base.functions.all_variables(include_internal=True)
            except:
                pass
    else:
        the_vars = None
    json_filename = None
    if the_vars is not None and len(the_vars):
        try:
            with tempfile.NamedTemporaryFile(mode='w', prefix="datemp", suffix='.json', delete=False, encoding='utf-8') as fp:
                fp.write(json.dumps(the_vars, sort_keys=True, indent=2))
                json_filename = fp.name
        except Exception as the_err:
            pass
    interview_path = docassemble.base.functions.interview_path()
    try:
        the_key = 'da:errornotification:' + str(ipaddress)
        existing = r.get(the_key)
        pipe = r.pipeline()
        pipe.set(the_key, 1)
        pipe.expire(the_key, 60)
        pipe.execute()
        if existing:
            return
    except:
        pass
    try:
        try:
            body = "There was an error in the " + app.config['APP_NAME'] + " application.\n\nThe error message was:\n\n" + err.__class__.__name__ + ": " + str(errmess)
            if trace is not None:
                body += "\n\n" + str(trace)
            if history is not None:
                body += "\n\n" + BeautifulSoup(history, "html.parser").get_text('\n')
            if referer is not None and referer != 'None':
                body += "\n\nThe referer URL was " + str(referer)
            elif interview_path is not None:
                body += "\n\nThe interview was " + str(interview_path)
            if email_address is not None:
                body += "\n\nThe user was " + str(email_address)
            html = "<html>\n  <body>\n    <p>There was an error in the " + app.config['APP_NAME'] + " application.</p>\n    <p>The error message was:</p>\n<pre>" + err.__class__.__name__ + ": " + str(errmess)
            if trace is not None:
                html += "\n\n" + str(trace)
            html += "</pre>\n"
            if history is not None:
                html += str(history)
            if referer is not None and referer != 'None':
                html += "<p>The referer URL was " + str(referer) + "</p>"
            elif interview_path is not None:
                body += "<p>The interview was " + str(interview_path) + "</p>"
            if email_address is not None:
                body += "<p>The user was " + str(email_address) + "</p>"
            if 'external hostname' in daconfig and daconfig['external hostname'] is not None:
                body += "<p>The external hostname was " + str(daconfig['external hostname']) + "</p>"
            html += "\n  </body>\n</html>"
            msg = Message(app.config['APP_NAME'] + " error: " + err.__class__.__name__, recipients=email_recipients, body=body, html=html)
            if json_filename:
                with open(json_filename, 'r', encoding='utf-8') as fp:
                    msg.attach('variables.json', 'application/json', fp.read())
            da_send_mail(msg)
        except Exception as zerr:
            logmessage(str(zerr))
            body = "There was an error in the " + app.config['APP_NAME'] + " application."
            html = "<html>\n  <body>\n    <p>There was an error in the " + app.config['APP_NAME'] + " application.</p>\n  </body>\n</html>"
            msg = Message(app.config['APP_NAME'] + " error: " + err.__class__.__name__, recipients=email_recipients, body=body, html=html)
            if json_filename:
                with open(json_filename, 'r', encoding='utf-8') as fp:
                    msg.attach('variables.json', 'application/json', fp.read())
            da_send_mail(msg)
    except:
        pass

def stash_data(data, expire):
    while True:
        key = random_alphanumeric(16)
        if r.get(key) is None:
            break
    secret = random_string(16)
    packed_data = encrypt_dictionary(data, secret)
    pipe = r.pipeline()
    pipe.set('da:stash:' + key, packed_data)
    pipe.expire('da:stash:' + key, expire)
    pipe.execute()
    return (key, secret)

def retrieve_stashed_data(key, secret, delete=False, refresh=False):
    packed_data = r.get('da:stash:' + key)
    if packed_data is None:
        return None
    try:
        data = decrypt_dictionary(packed_data.decode(), secret)
    except:
        return None
    if delete:
        r.delete('da:stash:' + key)
    elif refresh and isinstance(refresh, int) and refresh > 0:
        r.expire('da:stash:' + key, refresh)
    return data

def make_necessary_dirs():
    for path in (FULL_PACKAGE_DIRECTORY, UPLOAD_DIRECTORY, LOG_DIRECTORY): #PACKAGE_CACHE
        if not os.path.isdir(path):
            try:
                os.makedirs(path, exist_ok=True)
            except:
                sys.exit("Could not create path: " + path)
        if not os.access(path, os.W_OK):
            sys.exit("Unable to create files in directory: " + path)
    if not os.access(WEBAPP_PATH, os.W_OK):
        sys.exit("Unable to modify the timestamp of the WSGI file: " + WEBAPP_PATH)

make_necessary_dirs()

docassemble.base.functions.update_server(url_finder=get_url_from_file_reference,
                                         navigation_bar=navigation_bar,
                                         chat_partners_available=chat_partners_available,
                                         get_chat_log=get_current_chat_log,
                                         sms_body=sms_body,
                                         send_fax=da_send_fax,
                                         get_sms_session=get_sms_session,
                                         initiate_sms_session=initiate_sms_session,
                                         terminate_sms_session=terminate_sms_session,
                                         applock=applock,
                                         twilio_config=twilio_config,
                                         server_redis=r,
                                         server_redis_user=r_user,
                                         user_id_dict=user_id_dict,
                                         get_user_object=get_user_object,
                                         retrieve_emails=retrieve_emails,
                                         get_short_code=get_short_code,
                                         make_png_for_pdf=make_png_for_pdf,
                                         task_ready=task_ready,
                                         wait_for_task=wait_for_task,
                                         user_interviews=user_interviews,
                                         interview_menu=interview_menu,
                                         get_user_list=get_user_list,
                                         get_user_info=get_user_info,
                                         set_user_info=set_user_info,
                                         make_user_inactive=make_user_inactive,
                                         get_secret=get_secret,
                                         get_session_variables=get_session_variables,
                                         go_back_in_session=go_back_in_session,
                                         create_session=create_new_interview,
                                         set_session_variables=set_session_variables,
                                         get_privileges_list=get_privileges_list,
                                         add_privilege=add_privilege,
                                         remove_privilege=remove_privilege,
                                         add_user_privilege=add_user_privilege,
                                         remove_user_privilege=remove_user_privilege,
                                         create_user=create_user,
                                         file_set_attributes=file_set_attributes,
                                         file_user_access=file_user_access,
                                         file_privilege_access=file_privilege_access,
                                         fg_make_png_for_pdf=fg_make_png_for_pdf,
                                         fg_make_png_for_pdf_path=fg_make_png_for_pdf_path,
                                         fg_make_pdf_for_word_path=fg_make_pdf_for_word_path,
                                         get_question_data=get_question_data,
                                         fix_pickle_obj=fix_pickle_obj,
                                         main_page_parts=main_page_parts,
                                         SavedFile=SavedFile,
                                         path_from_reference=path_from_reference,
                                         button_class_prefix=app.config['BUTTON_STYLE'],
                                         write_answer_json=write_answer_json,
                                         read_answer_json=read_answer_json,
                                         delete_answer_json=delete_answer_json,
                                         variables_snapshot_connection=variables_snapshot_connection,
                                         get_referer=get_referer,
                                         stash_data=stash_data,
                                         retrieve_stashed_data=retrieve_stashed_data)

#docassemble.base.util.set_user_id_function(user_id_dict)
#docassemble.base.functions.set_generate_csrf(generate_csrf)
#docassemble.base.parse.set_url_finder(get_url_from_file_reference)
#docassemble.base.parse.set_url_for(url_for)
#APPLICATION_NAME = 'docassemble'

base_words = get_base_words()
title_documentation = get_title_documentation()
documentation_dict = get_documentation_dict()
base_name_info = get_name_info()

#docassemble.base.functions.set_chat_partners_available(chat_partners_available)

password_secret_key = daconfig.get('password secretkey', app.secret_key)

sys_logger = logging.getLogger('docassemble')
sys_logger.setLevel(logging.DEBUG)

LOGFORMAT = daconfig.get('log format', 'docassemble: ip=%(clientip)s i=%(yamlfile)s uid=%(session)s user=%(user)s %(message)s')

def add_log_handler():
    tries = 0
    while tries < 5:
        try:
            docassemble_log_handler = logging.FileHandler(filename=os.path.join(LOG_DIRECTORY, 'docassemble.log'))
        except PermissionError:
            time.sleep(1)
            next
        sys_logger.addHandler(docassemble_log_handler)
        break

add_log_handler()

if not in_celery:
    if LOGSERVER is None:
        docassemble.base.logger.set_logmessage(syslog_message_with_timestamp)
    else:
        docassemble.base.logger.set_logmessage(syslog_message)

def get_base_url():
    return re.sub(r'^(https?://[^/]+).*', r'\1', url_for('rootindex', _external=True))

def null_func(*pargs, **kwargs):
    logmessage("Null function called")
    return None

if in_celery:
    def illegal_worker_convert(*pargs, **kwargs):
        raise Exception("You cannot access the status of a background task from inside of a background task.")
    docassemble.base.functions.update_server(bg_action=null_func,
                                             #async_ocr=null_func,
                                             chord=null_func,
                                             ocr_page=null_func,
                                             ocr_finalize=null_func,
                                             worker_convert=illegal_worker_convert)
else:
    docassemble.base.functions.update_server(bg_action=docassemble.webapp.worker.background_action,
                                             #async_ocr=docassemble.webapp.worker.async_ocr,
                                             chord=docassemble.webapp.worker.chord,
                                             ocr_page=docassemble.webapp.worker.ocr_page,
                                             ocr_dummy=docassemble.webapp.worker.ocr_dummy,
                                             ocr_finalize=docassemble.webapp.worker.ocr_finalize,
                                             worker_convert=docassemble.webapp.worker.convert)

pg_ex = {}

def define_examples():
    if 'encoded_example_html' in pg_ex:
        return
    example_html = []
    example_html.append('        <div class="col-md-2">\n          <h5 class="mb-1">Example blocks</h5>')
    pg_ex['pg_first_id'] = []
    data_dict = {}
    make_example_html(get_examples(), pg_ex['pg_first_id'], example_html, data_dict)
    example_html.append('        </div>')
    example_html.append('        <div class="col-md-4 da-example-source-col"><h5 class="mb-1">' + word('Source') + '<a href="#" tabindex="0" class="dabadge btn btn-success da-example-copy">' + word("Insert") + '</a></h5><div id="da-example-source-before" class="dainvisible"></div><div id="da-example-source"></div><div id="da-example-source-after" class="dainvisible"></div><div><a tabindex="0" class="da-example-hider" id="da-show-full-example">' + word("Show context of example") + '</a><a tabindex="0" class="da-example-hider dainvisible" id="da-hide-full-example">' + word("Hide context of example") + '</a></div></div>')
    example_html.append('        <div class="col-md-6"><h5 class="mb-1">' + word("Preview") + '<a href="#" target="_blank" class="dabadge btn btn-primary da-example-documentation da-example-hidden" id="da-example-documentation-link">' + word("View documentation") + '</a></h5><a href="#" target="_blank" id="da-example-image-link"><img title=' + json.dumps(word("Click to try this interview")) + ' class="da-example-screenshot" id="da-example-image"></a></div>')
    pg_ex['encoded_data_dict'] = safeid(json.dumps(data_dict))
    pg_ex['encoded_example_html'] = Markup("\n".join(example_html))

if LooseVersion(min_system_version) > LooseVersion(daconfig['system version']):
    version_warning = word("A new docassemble system version is available.  If you are using Docker, install a new Docker image.")
else:
    version_warning = None

class AdminInterview:
    def can_use(self):
        if self.require_login and current_user.is_anonymous:
            return False
        if self.roles is None:
            return True
        if current_user.is_anonymous:
            if 'anonymous' in self.roles:
                return True
            return False
        if current_user.has_roles(self.roles):
            return True
        return False
    def get_title(self, language):
        if isinstance(self.title, str):
            return word(self.title, language=language)
        return self.title.get(language, word(self.title, language=language))

def set_admin_interviews():
    admin_interviews = []
    if 'administrative interviews' in daconfig:
        if isinstance(daconfig['administrative interviews'], list):
            for item in daconfig['administrative interviews']:
                if isinstance(item, dict):
                    if 'interview' in item and isinstance(item['interview'], str):
                        try:
                            interview = docassemble.base.interview_cache.get_interview(item['interview'])
                        except:
                            sys.stderr.write("interview " + item['interview'] + " in administrative interviews did not exist" + "\n")
                            continue
                        if 'title' in item:
                            the_title = item['title']
                        else:
                            the_title = interview.consolidated_metadata.get('short title', interview.consolidated_metadata.get('title', None))
                            if the_title is None:
                                sys.stderr.write("interview in administrative interviews needs to be given a title" + "\n")
                                continue
                        admin_interview = AdminInterview()
                        admin_interview.interview = item['interview']
                        if isinstance(the_title, (str, dict)):
                            if isinstance(the_title, dict):
                                fault = False
                                for key, val in the_title.items():
                                    if not (isinstance(key, str) and isinstance(val, str)):
                                        fault = True
                                        break
                                if fault:
                                    sys.stderr.write("title of administrative interviews item must be a string or a dictionary with keys and values that are strings" + "\n")
                                    continue
                            admin_interview.title = the_title
                        else:
                            sys.stderr.write("title of administrative interviews item must be a string or a dictionary" + "\n")
                            continue
                        if 'required privileges' not in item:
                            roles = set()
                            for metadata in interview.metadata:
                                if 'required privileges for listing' in metadata:
                                    roles = set()
                                    privs = metadata['required privileges for listing']
                                    if isinstance(privs, list):
                                        for priv in privs:
                                            if isinstance(priv, str):
                                                roles.add(priv)
                                    elif isinstance(privs, str):
                                        roles.add(privs)
                                elif 'required privileges' in metadata:
                                    roles = set()
                                    privs = metadata['required privileges']
                                    if isinstance(privs, list):
                                        for priv in privs:
                                            if isinstance(priv, str):
                                                roles.add(priv)
                                    elif isinstance(privs, str):
                                        roles.add(privs)
                            if len(roles) > 0:
                                item['required privileges'] = list(roles)
                        if 'required privileges' in item:
                            fault = False
                            if isinstance(item['required privileges'], list):
                                for rolename in item['required privileges']:
                                    if not isinstance(rolename, str):
                                        fault = True
                                        break
                            else:
                                fault = True
                            if fault:
                                sys.stderr.write("required privileges in administrative interviews item must be a list of strings" + "\n")
                                admin_interview.roles = None
                            else:
                                admin_interview.roles = item['required privileges']
                        else:
                            admin_interview.roles = None
                        admin_interview.require_login = False
                        for metadata in interview.metadata:
                            if 'require login' in metadata:
                                admin_interview.require_login = bool(metadata['require login'])
                        admin_interviews.append(admin_interview)
                    else:
                        sys.stderr.write("item in administrative interviews must contain a valid interview name" + "\n")
                else:
                    sys.stderr.write("item in administrative interviews is not a dict" + "\n")
        else:
            sys.stderr.write("administrative interviews is not a list" + "\n")
    return admin_interviews

def encrypt_api_key(key):
    return codecs.encode(hashlib.pbkdf2_hmac('sha256', bytearray(key, 'utf-8'), bytearray(app.secret_key, encoding='utf-8'), 100000), 'base64').decode().strip()

def fix_api_key(match):
    return 'da:apikey:userid:' + match.group(1) + ':key:' + encrypt_api_key(match.group(2)) + ':info'

def fix_api_keys():
    to_delete = []
    for rkey in r.keys('da:api:userid:*:key:*:info'):
        try:
            rkey = rkey.decode()
        except:
            continue
        try:
            info = json.loads(r.get(rkey).decode())
            assert isinstance(info, dict)
        except:
            to_delete.append(rkey)
            continue
        info['last_four'] = re.sub(r'da:api:userid:.*:key:.*(....):info', r'\1', rkey)
        new_rkey = re.sub(r'da:api:userid:(.*):key:(.*):info', fix_api_key, rkey)
        r.set(new_rkey, json.dumps(info))
        to_delete.append(rkey)
    for rkey in to_delete:
        r.delete(rkey)

def random_social():
    while True:
        new_social = 'local$' + random_alphanumeric(32)
        existing_user = db.session.execute(select(UserModel).filter_by(social_id=new_social)).first()
        if existing_user:
            continue
        break
    return new_social

class TestContext:
    def __init__(self, package):
        self.package = package

    def __enter__(self):
        url_root = daconfig.get('url root', 'http://localhost') + daconfig.get('root', '/')
        url = url_root + 'interview'
        self.app_context = app.app_context()
        self.app_context.push()
        self.test_context = app.test_request_context(base_url=url_root, path=url)
        self.test_context.push()
        login_as_admin(url, url_root)
        docassemble.base.functions.this_thread.current_package = self.package
        docassemble.base.functions.this_thread.current_info.update(dict(yaml_filename=self.package + ':data/questions/test.yml'))
        return self

    def __exit__(self, exc_type, exc_value, exc_traceback):
        current_app.login_manager._update_request_context_with_user()
        self.test_context.pop()
        self.app_context.pop()

def initialize():
    global global_css
    global global_js
    with app.app_context():
        app.user_manager.random_social = random_social
        url_root = daconfig.get('url root', 'http://localhost') + daconfig.get('root', '/')
        url = url_root + 'interview'
        with app.test_request_context(base_url=url_root, path=url):
            if 'bootstrap theme' in daconfig and daconfig['bootstrap theme']:
                try:
                    app.config['BOOTSTRAP_THEME'] = get_url_from_file_reference(daconfig['bootstrap theme'])
                    assert isinstance(app.config['BOOTSTRAP_THEME'], str)
                except:
                    app.config['BOOTSTRAP_THEME'] = None
                    sys.stderr.write("error loading bootstrap theme\n")
            else:
                app.config['BOOTSTRAP_THEME'] = None
            if 'global css' in daconfig:
                for fileref in daconfig['global css']:
                    try:
                        global_css_url = get_url_from_file_reference(fileref)
                        assert isinstance(global_css_url, str)
                        global_css += "\n" + '    <link href="' + global_css_url + '" rel="stylesheet">'
                    except:
                        sys.stderr.write("error loading global css: " + repr(fileref) + "\n")
            if 'global javascript' in daconfig:
                for fileref in daconfig['global javascript']:
                    try:
                        global_js_url = get_url_from_file_reference(fileref)
                        assert isinstance(global_js_url, str)
                        global_js += "\n" + '    <script src="' + global_js_url + '"></script>'
                    except:
                        sys.stderr.write("error loading global js: " + repr(fileref) + "\n")
            if 'raw global css' in daconfig and daconfig['raw global css']:
                global_css += "\n" + str(daconfig['raw global css'])
            if 'raw global javascript' in daconfig and daconfig['raw global javascript']:
                global_js += "\n" + str(daconfig['raw global javascript'])
            app.config['GLOBAL_CSS'] = global_css
            app.config['GLOBAL_JS'] = global_js
            app.config['PARTS'] = page_parts
            app.config['ADMIN_INTERVIEWS'] = set_admin_interviews()
            app.config['ENABLE_PLAYGROUND'] = daconfig.get('enable playground', True)
            app.config['ALLOW_UPDATES'] = daconfig.get('allow updates', True)
            try:
                if 'image' in daconfig['social'] and isinstance(daconfig['social']['image'], str):
                    daconfig['social']['image'] = get_url_from_file_reference(daconfig['social']['image'], _external=True)
                    if daconfig['social']['image'] is None:
                        del daconfig['social']['image']
                for key, subkey in (('og', 'image'), ('twitter', 'image')):
                    if key in daconfig['social'] and isinstance(daconfig['social'][key], dict) and subkey in daconfig['social'][key] and isinstance(daconfig['social'][key][subkey], str):
                        daconfig['social'][key][subkey] = get_url_from_file_reference(daconfig['social'][key][subkey], _external=True)
                        if daconfig['social'][key][subkey] is None:
                            del daconfig['social'][key][subkey]
            except:
                sys.stderr.write("Error converting social image references")
            interviews_to_load = daconfig.get('preloaded interviews', None)
            if isinstance(interviews_to_load, list):
                for yaml_filename in daconfig['preloaded interviews']:
                    try:
                        docassemble.base.interview_cache.get_interview(yaml_filename)
                    except:
                        pass
            if app.config['ENABLE_PLAYGROUND']:
                obtain_lock('init', 'init')
                try:
                    copy_playground_modules()
                except Exception as err:
                    sys.stderr.write("There was an error copying the playground modules: " + err.__class__.__name__ + "\n")
                write_pypirc()
                release_lock('init', 'init')
            try:
                macro_path = daconfig.get('libreoffice macro file', '/var/www/.config/libreoffice/4/user/basic/Standard/Module1.xba')
                if os.path.isfile(macro_path) and os.path.getsize(macro_path) != 7167:
                    sys.stderr.write("Removing " + macro_path + " because it is out of date\n")
                    os.remove(macro_path)
                # else:
                #     sys.stderr.write("File " + macro_path + " is missing or has the correct size\n")
            except Exception as err:
                sys.stderr.write("Error was " + err.__class__.__name__ + ' ' + str(err) + "\n")
            fix_api_keys()
            import_necessary(url, url_root)

initialize()

if __name__ == "__main__":
    app.run()
