#!/bin/bash

"export" "LD_LIBRARY_PATH_ORIG=${LD_LIBRARY_PATH}"
"export" "LD_LIBRARY_PATH="
"export" "PYTHONPATH_ORIG=${PYTHONPATH}"
"export" "PYTHONPATH=${PANDA_PYTHONPATH}"
"export" "PYTHONHOME_ORIG=${PYTHONHOME}"
"unset" "PYTHONHOME"
"exec" "/usr/bin/python" "-u" "-Wignore" "$0" "$@"

import os
import re
import sys
import copy
import shutil
import atexit
import argparse
import random
import types
import pickle
import json

####################################################################

# error code
EC_Config    = 10
EC_CMT       = 20
EC_Extractor = 30
EC_Dataset   = 40
EC_Post      = 50
EC_Archive   = 60
EC_Split     = 70
EC_MyProxy   = 80
EC_Submit    = 90

usage = """pathena [options] <jobOption1.py> [<jobOption2.py> [...]]

pathena --help' prints a summary of the options

  HowTo is available at https://twiki.cern.ch/twiki/bin/view/PanDA/PandaAthena"""


# command-line parameters
optP = argparse.ArgumentParser(usage=usage,conflict_handler="resolve")
# special options
optP.add_argument('--version',action='store_const',const=True,dest='version',default=False,
                help='Displays version')
optP.add_argument('--split', action='store', dest='split',  default=-1,
                type=int,    help='Number of sub-jobs to be generated. This option is the same as --nJobs')
optP.add_argument('--nJobs', action='store', dest='nJobs',  default=-1,
                type=int,    help='Number of sub-jobs to be generated. This option is the same as --split')
optP.add_argument('--nFilesPerJob', action='store', dest='nFilesPerJob',  default=-1, type=int, help='Number of files on which each sub-job runs')
optP.add_argument('--nEventsPerJob', action='store', dest='nEventsPerJob',  default=-1,
                type=int,    help='Number of events per subjob. This info is used mainly for job splitting. If you run on MC datasets, the total number of subjobs is nEventsPerFile*nFiles/nEventsPerJob. For data, the number of events for each file is retrieved from AMI and subjobs are created accordingly. Note that if you run transformations you need to explicitly specify maxEvents or something in --trf to set the number of events processed in each subjob. If you run normal jobOption files, evtMax and skipEvents in appMgr are automatically set on WN.')
optP.add_argument('--nEventsPerFile', action='store', dest='nEventsPerFile',  default=0,
                type=int,    help='Number of events per file')
optP.add_argument('--nGBPerJob',action='store',dest='nGBPerJob',default=-1, help='Instantiate one sub job per NGBPERJOB GB of input files. --nGBPerJob=MAX sets the size to the default maximum value')
optP.add_argument('--nGBPerMergeJob',action='store',dest='nGBPerMergeJob',default=-1, help='Instantiate one merge job per NGBPERMERGEJOB GB of pre-merged files')
optP.add_argument('--site', action='store', dest='site',  default="AUTO",
                type=str,    help='Site name where jobs are sent. If omitted, jobs are automatically sent to sites where input is available. A comma-separated list of sites can be specified (e.g. siteA,siteB,siteC), so that best sites are chosen from the given site list. If AUTO is appended at the end of the list (e.g. siteA,siteB,siteC,AUTO), jobs are sent to any sites if input is not found in the previous sites')
optP.add_argument('--athenaTag',action='store',dest='athenaTag',default='',type=str,
                help='Use differnet version of Athena on remote WN. By defualt the same version which you are locally using is set up on WN. e.g., --athenaTag=AtlasProduction,14.2.24.3')
optP.add_argument('--inDS',  action='store', dest='inDS',  default='',
                type=str, help='Input dataset names. wildcard and/or comma can be used to concatenate multiple datasets')
optP.add_argument('--inDsTxt',action='store',dest='inDsTxt',default='',
                type=str, help='a text file which contains the list of datasets to run over. newlines are replaced by commas and the result is set to --inDS. lines starting with # are ignored')
optP.add_argument('--inOutDsJson', action='store', dest='inOutDsJson', default='',
                  help="A json file to specify input and output datasets for bulk submission. It contains a json dump of [{'inDS': a comma-concatenated input dataset names, 'outDS': output dataset name}, ...]")
optP.add_argument('--minDS',  action='store', dest='minDS',  default='',
                type=str, help='Dataset name for minimum bias stream')
optP.add_argument('--nMin',  action='store', dest='nMin',  default=-1,
                type=int, help='Number of minimum bias files per sub job')
optP.add_argument('--lowMinDS',  action='store', dest='lowMinDS',  default='',
                type=str, help='Dataset name for low pT minimum bias stream')
optP.add_argument('--nLowMin',  action='store', dest='nLowMin',  default=-1,
                type=int, help='Number of low pT minimum bias files per job')
optP.add_argument('--highMinDS',  action='store', dest='highMinDS',  default='',
                type=str, help='Dataset name for high pT minimum bias stream')
optP.add_argument('--nHighMin',  action='store', dest='nHighMin',  default=-1,
                type=int, help='Number of high pT minimum bias files per job')
optP.add_argument('--randomMin',action='store_const',const=True,dest='randomMin',default=False,
                help='randomize files in minimum bias dataset')
optP.add_argument('--cavDS',  action='store', dest='cavDS',  default='',
                type=str, help='Dataset name for cavern stream')
optP.add_argument('--nCav',  action='store', dest='nCav',  default=-1,
                type=int, help='Number of cavern files per job')
optP.add_argument('--randomCav',action='store_const',const=True,dest='randomCav',default=False,
                help='randomize files in cavern dataset')
optP.add_argument('--goodRunListXML', action='store', dest='goodRunListXML', default='',
                type=str, help='Good Run List XML which will be converted to datasets by AMI')
optP.add_argument('--goodRunListDataType', action='store', dest='goodRunDataType', default='',
                type=str, help='specify data type when converting Good Run List XML to datasets, e.g, AOD (default)')
optP.add_argument('--goodRunListProdStep', action='store', dest='goodRunProdStep', default='',
                type=str, help='specify production step when converting Good Run List to datasets, e.g, merge (default)')
optP.add_argument('--goodRunListDS', action='store', dest='goodRunListDS', default='',
                type=str, help='A comma-separated list of pattern strings. Datasets which are converted from Good Run List XML will be used when they match with one of the pattern strings. Either \ or "" is required when a wild-card is used. If this option is omitted all datasets will be used')
optP.add_argument('--eventPickEvtList',action='store',dest='eventPickEvtList',default='',
                type=str, help='a file name which contains a list of runs/events for event picking')
optP.add_argument('--eventPickDataType',action='store',dest='eventPickDataType',default='',
                type=str, help='type of data for event picking. one of AOD,ESD,RAW')
optP.add_argument('--ei_api',action='store',dest='ei_api',default='',
                type=str, help='flag to signalise mc in event picking')
optP.add_argument('--eventPickStreamName',action='store',dest='eventPickStreamName',default='',
                type=str, help='stream name for event picking. e.g., physics_CosmicCaloEM')
optP.add_argument('--eventPickDS',action='store',dest='eventPickDS',default='',
                type=str, help='A comma-separated list of pattern strings. Datasets which are converted from the run/event list will be used when they match with one of the pattern strings. Either \ or "" is required when a wild-card is used. e.g., data\*')
optP.add_argument('--eventPickAmiTag',action='store',dest='eventPickAmiTag',default='',
                type=str, help='AMI tag used to match TAG collections names. This option is required when you are interested in older data than the latest one. Either \ or "" is required when a wild-card is used. e.g., f2\*')
optP.add_argument('--eventPickWithGUID',action='store_const',const=True,dest='eventPickWithGUID',default=False,
                help='Using GUIDs together with run and event numbers in eventPickEvtList to skip event lookup')
optP.add_argument('--sameSecRetry', action='store_const',const=False,dest='sameSecRetry',default=True,
                help="Use the same secondary input files when jobs are retried")
optP.add_argument('--express', action='store_const',const=True,dest='express',default=False,
                help="Send the job using express quota to have higher priority. The number of express subjobs in the queue and the total execution time used by express subjobs are limited (a few subjobs and several hours per day, respectively). This option is intended to be used for quick tests before bulk submission. Note that buildXYZ is not included in quota calculation. If this option is used when quota has already exceeded, the panda server will ignore the option so that subjobs have normal priorities. Also, if you submit 1 buildXYZ and N runXYZ subjobs when you only have quota of M (M < N),  only the first M runXYZ subjobs will have higher priorities")
optP.add_argument('--debugMode', action='store_const',const=True,dest='debugMode',default=False,
                help="Send the job with the debug mode on. If this option is specified the subjob will send stdout to the panda monitor every 5 min. The number of debug subjobs per user is limited. When this option is used and the quota has already exceeded, the panda server supresses the option so that subjobs will run without the debug mode. If you submit multiple subjobs in a single job, only the first subjob will set the debug mode on. Note that you can turn the debug mode on/off by using pbook after jobs are submitted" )
optP.add_argument('--addNthFieldOfInDSToLFN',action='store',dest='addNthFieldOfInDSToLFN',default='',type=str,
                help="A middle name is added to LFNs of output files when they are produced from one dataset in the input container or input dataset list. The middle name is extracted from the dataset name. E.g., if --addNthFieldOfInDSToLFN=2 and the dataset name is data10_7TeV.00160387.physics_Muon..., 00160387 is extracted and LFN is something like user.hoge.TASKID.00160387.blah. Concatenate multiple field numbers with commas if necessary, e.g., --addNthFieldOfInDSToLFN=2,6.")
optP.add_argument('--addNthFieldOfInFileToLFN',action='store',dest='addNthFieldOfInFileToLFN',default='',type=str,
                help="A middle name is added to LFNs of output files similarly as --addNthFieldOfInDSToLFN, but strings are extracted from input file names")
optP.add_argument('--useAMIEventLevelSplit',action='store_const',const=True,dest='useAMIEventLevelSplit',default=None,
                help="retrive the number of events per file from AMI to split the job using --nEventsPerJob")
optP.add_argument('--appendStrToExtStream',action='store_const',const=True,dest='appendStrToExtStream',default=False,
                help='append the first part of filenames to extra stream names for --individualOutDS. E.g., if this option is used together with --individualOutDS, %%OUT.AOD.pool.root will be contained in an EXT0_AOD dataset instead of an EXT0 dataset')
optP.add_argument('--mergeOutput', action='store_const', const=True, dest='mergeOutput', default=False,
                help="merge output files")
optP.add_argument('--mergeScript',action='store',dest='mergeScript',default='',type=str,
                help='Specify user-defied script or execution string for output merging')
optP.add_argument('--useCommonHalo', action='store_const', const=False, dest='useCommonHalo',  default=True,
                help="use an integrated DS for BeamHalo")
optP.add_argument('--beamHaloDS',  action='store', dest='beamHaloDS',  default='',
                type=str, help='Dataset name for beam halo')
optP.add_argument('--beamHaloADS',  action='store', dest='beamHaloADS',  default='',
                type=str, help='Dataset name for beam halo A-side')
optP.add_argument('--beamHaloCDS',  action='store', dest='beamHaloCDS',  default='',
                type=str, help='Dataset name for beam halo C-side')
optP.add_argument('--nBeamHalo',  action='store', dest='nBeamHalo',  default=-1,
                type=int, help='Number of beam halo files per sub job')
optP.add_argument('--nBeamHaloA',  action='store', dest='nBeamHaloA',  default=-1,
                type=int, help='Number of beam halo files for A-side per sub job')
optP.add_argument('--nBeamHaloC',  action='store', dest='nBeamHaloC',  default=-1,
                type=int, help='Number of beam halo files for C-side per sub job')
optP.add_argument('--useCommonGas', action='store_const', const=False, dest='useCommonGas',  default=True,
                help="use an integrated DS for BeamGas")
optP.add_argument('--beamGasDS',  action='store', dest='beamGasDS',  default='',
                type=str, help='Dataset name for beam gas')
optP.add_argument('--beamGasHDS',  action='store', dest='beamGasHDS',  default='',
                type=str, help='Dataset name for beam gas Hydrogen')
optP.add_argument('--beamGasCDS',  action='store', dest='beamGasCDS',  default='',
                type=str, help='Dataset name for beam gas Carbon')
optP.add_argument('--beamGasODS',  action='store', dest='beamGasODS',  default='',
                type=str, help='Dataset name for beam gas Oxygen')
optP.add_argument('--nBeamGas',  action='store', dest='nBeamGas',  default=-1,
                type=int, help='Number of beam gas files per sub job')
optP.add_argument('--nBeamGasH',  action='store', dest='nBeamGasH',  default=-1,
                type=int, help='Number of beam gas files for Hydrogen per sub job')
optP.add_argument('--nBeamGasC',  action='store', dest='nBeamGasC',  default=-1,
                type=int, help='Number of beam gas files for Carbon per sub job')
optP.add_argument('--nBeamGasO',  action='store', dest='nBeamGasO',  default=-1,
                type=int, help='Number of beam gas files for Oxygen per sub job')
optP.add_argument('--outDS', action='store', dest='outDS', default='',
                type=str, help='Name of an output dataset. OUTDS will contain all output files')
optP.add_argument('--destSE',action='store', dest='destSE',default='',
                type=str, help='Destination strorage element')
optP.add_argument('--nFiles', '--nfiles', action='store', dest='nfiles',  default=0,
                type=int,    help='Use an limited number of files in the input dataset')
optP.add_argument('-v', action='store_const', const=True, dest='verbose',  default=False,
                help='Verbose')
optP.add_argument('--noEmail', action='store_const', const=True, dest='noEmail',  default=False,
                help='Suppress email notification')
optP.add_argument('--update', action='store_const', const=True, dest='update',  default=False,
                help='Update panda-client to the latest version')
optP.add_argument('--noBuild', action='store_const', const=True, dest='noBuild',  default=False,
                help='Skip buildJob')
optP.add_argument('--bulkSubmission', action='store_const', const=True, dest='bulkSubmission', default=False,
                help='Bulk submit tasks. When this option is used, --inOutDsJson is required while --inDS and --outDS are ignored. It is possible to use %%DATASET_IN and %%DATASET_OUT in --trf which are replaced with actual dataset names when tasks are submitted, and %%BULKSEQNUMBER which is replaced with a sequential number of tasks in the bulk submission')
optP.add_argument('--noCompile', action='store_const',const=True,dest='noCompile',default=False,
                help='Just upload a tarball in the build step to avoid the tighter size limit imposed by --noBuild. The tarball contains binaries compiled on your local computer, so that compilation is skipped in the build step on remote WN')
optP.add_argument('--noOutput', action='store_const', const=True, dest='noOutput',  default=False,
                help='Send job even if there is no output file')
optP.add_argument('--noRandom', action='store_const', const=True, dest='norandom',  default=False,
                help='Enter random seeds manually')
optP.add_argument('--useAMIAutoConf',action='store_const',const=True,dest='useAMIAutoConf',default=False,
                help='Use AMI for AutoConfiguration')
optP.add_argument('--memory', action='store', dest='memory',  default=-1,
                type=int,    help='Required memory size in MB. e.g., for 1GB --memory 1024')
optP.add_argument('--nCore', action='store', dest='nCore', default=-1,
                type=int,    help='The number of CPU cores. Note that the system distinguishes only nCore=1 and nCore>1. This means that even if you set nCore=2 jobs can go to sites with nCore=8 and your application must use the 8 cores there. The number of available cores is defined in an environment variable, $ATHENA_PROC_NUMBER, on WNs. Your application must check the env variable when starting up to dynamically change the number of cores')
optP.add_argument('--nThreads', action='store', dest='nThreads', default=-1,
                type=int,    help='The number of threads for AthenaMT. If this option is set to larger than 1, Athena is executed with --threads=$ATHENA_PROC_NUMBER at sites which have nCore>1. This means that even if you set nThreads=2 jobs can go to sites with nCore=8 and your application will use the 8 cores there')
optP.add_argument('--forceStaged', action='store_const', const=True, dest='forceStaged', default=False,
                help='Force files from primary DS to be staged to local disk, even if direct-access is possible')
optP.add_argument('--maxCpuCount', action='store', dest='maxCpuCount', default=0, type=int,
                help='Required CPU count in seconds. Mainly to extend time limit for looping job detection')
optP.add_argument('--official', action='store_const', const=True, dest='official',  default=False,
                help='Produce official dataset')
optP.add_argument('--unlimitNumOutputs', action='store_const', const=True, dest='unlimitNumOutputs',  default=False,
                help='Remove the limit on the number of outputs. Note that having too many outputs per job causes a severe load on the system. You may be banned if you carelessly use this option') 
optP.add_argument('--descriptionInLFN',action='store',dest='descriptionInLFN',default='',
                help='LFN is user.nickname.jobsetID.something (e.g. user.harumaki.12345.AOD._00001.pool) by default. This option allows users to put a description string into LFN. i.e., user.nickname.jobsetID.description.something')
optP.add_argument('--extFile', action='store', dest='extFile',  default='',
                help='pathena exports files with some special extensions (.C, .dat, .py .xml) in the current directory. If you want to add other files, specify their names, e.g., data1.root,data2.doc')
optP.add_argument('--excludeFile',action='store',dest='excludeFile',default='',
                help='specify a comma-separated string to exclude files and/or directories when gathering files in local working area. Either \ or "" is required when a wildcard is used. e.g., doc,\*.C')
optP.add_argument('--extOutFile', action='store', dest='extOutFile',  default='',
                help='A comma-separated list of extra output files which cannot be extracted automatically. Either \ or "" is required when a wildcard is used. e.g., output1.txt,output2.dat,JiveXML_\*.xml')
optP.add_argument('--supStream', action='store', dest='supStream',  default='',
                help='suppress some output streams. Either \ or "" is required when a wildcard is used. e.g., ESD,TAG,GLOBAL,StreamDESD\* ')
optP.add_argument('--gluePackages', action='store', dest='gluePackages',  default='',
                help='list of glue packages which pathena cannot find due to empty i686-slc4-gcc34-opt. e.g., External/AtlasHepMC,External/Lhapdf')
optP.add_argument('--allowNoOutput',action='store',dest='allowNoOutput',default='',type=str,
                help='A comma-separated list of regexp patterns. Output files are allowed not to be produced if their filenames match with one of regexp patterns. Jobs go to finished even if they are not produced on WN')
optP.add_argument('--excludedSite', action='append', dest='excludedSite',  default=[],
                help="list of sites which are not used for site section, e.g., ANALY_ABC,ANALY_XYZ")
optP.add_argument('--noSubmit', action='store_const', const=True, dest='noSubmit',  default=False,
                help="Don't submit jobs")
optP.add_argument('--prodSourceLabel', action='store', dest='prodSourceLabel',  default='',
                help="set prodSourceLabel")
optP.add_argument('--processingType', action='store', dest='processingType',  default='pathena',
                help="set processingType")
optP.add_argument('--workingGroup', action='store', dest='workingGroup',  default=None,
                help="set workingGroup")
optP.add_argument('--generalInput', action='store_const', const=True, dest='generalInput',  default=False,
                help='Read input files with general format except POOL,ROOT,ByteStream')
optP.add_argument('--tmpDir', action='store', dest='tmpDir', default='',
                type=str, help='Temporary directory in which an archive file is created')
optP.add_argument('--shipInput', action='store_const', const=True, dest='shipinput',  default=False,
                help='Ship input files to remote WNs')
optP.add_argument('--disableAutoRetry',action='store_const',const=True,dest='disableAutoRetry',default=False,
                help='disable automatic job retry on the server side')
optP.add_argument('--fileList', action='store', dest='filelist', default='',
                type=str, help='List of files in the input dataset to be run')
optP.add_argument('--dbRelease', action='store', dest='dbRelease', default='',
                type=str, help='DBRelease or CDRelease (DatasetName:FileName). e.g., ddo.000001.Atlas.Ideal.DBRelease.v050101:DBRelease-5.1.1.tar.gz. If --dbRelease=LATEST, the latest DBRelease is used')
optP.add_argument('--addPoolFC', action='store', dest='addPoolFC',  default='',
                help="file names to be inserted into PoolFileCatalog.xml except input files. e.g., MyCalib1.root,MyGeom2.root")
optP.add_argument('--inputFileList', action='store', dest='inputFileList', default='',
                type=str, help='name of file which contains a list of files to be run in the input dataset')
optP.add_argument('--voms', action='store', dest='vomsRoles',  default=None, type=str,
                help="generate proxy with paticular roles. e.g., atlas:/atlas/ca/Role=production,atlas:/atlas/fr/Role=pilot")
optP.add_argument('--useNextEvent', action='store_const', const=True, dest='useNextEvent',  default=False,
                help="Set this option if your jobO uses theApp.nextEvent(), e.g. for G4. Note that this option is not required when you run transformations using --trf")
optP.add_argument('--trf', action='store', dest='trf',  default=False,
                help='run transformation, e.g. --trf "csc_atlfast_trf.py %%IN %%OUT.AOD.root %%OUT.ntuple.root -1 0"')
optP.add_argument('--spaceToken', action='store', dest='spaceToken', default='',
                type=str, help='spacetoken for outputs. e.g., ATLASLOCALGROUPDISK')
optP.add_argument('--notSkipMissing', action='store_const', const=True, dest='notSkipMissing',  default=False,
                help='If input files are not read from SE, they will be skipped by default. This option disables the functionality')
optP.add_argument('--forceDirectIO', action='store_const', const=True, dest='forceDirectIO', default=False,
                help="Use directIO if directIO is available at the site ")
optP.add_argument('--expertOnly_skipScout', action='store_const',const=True,dest='skipScout',default=False,
                help=argparse.SUPPRESS)
optP.add_argument('--respectSplitRule', action='store_const',const=True,dest='respectSplitRule',default=False,
                help="force scout jobs to follow split rules like nGBPerJob")
optP.add_argument('--devSrv', action='store_const', const=True, dest='devSrv',  default=False,
                help="Please don't use this option. Only for developers to use the dev panda server")
optP.add_argument('--intrSrv', action='store_const', const=True, dest='intrSrv',  default=False,
                help="Please don't use this option. Only for developers to use the intr panda server")
optP.add_argument('--inputType', action='store', dest='inputType', default='',
                type=str, help='A regular expression pattern. Only files matching with the pattern in input dataset are used')
optP.add_argument('--outTarBall', action='store', dest='outTarBall', default='',
                type=str, help='Save a gzipped tarball of local files which is the input to buildXYZ')
optP.add_argument('--inTarBall', action='store', dest='inTarBall', default='',
                type=str, help='Use a gzipped tarball of local files as input to buildXYZ. Generall the tarball is created by using --outTarBall')
optP.add_argument('--outRunConfig', action='store', dest='outRunConfig', default='',
                type=str, help='Save extracted config information to a local file')
optP.add_argument('--inRunConfig', action='store', dest='inRunConfig', default='',
                type=str, help='Use a saved config information to skip config extraction')
optP.add_argument('--pfnList', action='store', dest='pfnList', default='',
                type=str, help='Name of file which contains a list of input PFNs. Those files can be un-registered in DDM')
optP.add_argument('--cmtConfig', action='store', dest='cmtConfig', default=None,
                type=str, help='CMTCONFIG=i686-slc5-gcc43-opt is used on remote worker-node by default even if you use another CMTCONFIG locally. This option allows you to use another CMTCONFIG remotely. e.g., --cmtConfig x86_64-slc5-gcc43-opt.')
optP.add_argument('--allowTaskDuplication',action='store_const',const=True,dest='allowTaskDuplication',default=False,
                help="As a general rule each task has a unique outDS and history of file usage is recorded per task. This option allows multiple tasks to contribute to the same outDS. Typically useful to submit a new task with the outDS which was used by another broken task. Use this option very carefully at your own risk, since file duplication happens when the second task runs on the same input which the first task successfully processed")
optP.add_argument('--skipFilesUsedBy', action='store', dest='skipFilesUsedBy', default='',
                type=str, help='A comma-separated list of TaskIDs. Files used by those tasks are skipped when running a new task')
optP.add_argument('--maxAttempt', action='store', dest='maxAttempt', default=-1,
                type=int, help='Maximum number of reattempts for each job (3 by default and not larger than 50)')
optP.add_argument('--containerImage', action='store', dest='containerImage', default='',
                type=str, help="Name of a container image")
# athena options
optP.add_argument('-c',action='store',dest='singleLine',type=str,default='',metavar='COMMAND',
                help='One-liner, runs before any jobOs')
optP.add_argument('-p',action='store',dest='preConfig',type=str,default='',metavar='BOOTSTRAP',
                help='location of bootstrap file')
optP.add_argument('-s',action='store_const',const=True,dest='codeTrace',default=False,
                help='show printout of included files')
optP.add_argument('--queueData', action='store', dest='queueData', default='',
                type=str, help="Please don't use this option. Only for developers")
optP.add_argument('--useNewCode',action='store_const',const=True,dest='useNewCode',default=False,
                help='When task are resubmitted with the same outDS, the original souce code is used to re-run on failed/unprocessed files. This option uploads new source code so that jobs will run with new binaries')
optP.add_argument('--loadJson', action='store', dest='loadJson',default=None,
                  help="Read command-line parameters from a json file which contains a dict of {parameter: value}. Arguemnts for Athena can be specified as {'atehna_args': [...,]}")
optP.add_argument('--dumpJson', action='store', dest='dumpJson', default=None,
                  help='Dump all command-line parameters and submission result such as returnCode, returnOut, jediTaskID, and bulkSeqNumber if --bulkSubmission is used, to a json file')
optP.add_argument('--priority', action='store', dest='priority',  default=None, type=int,
                  help='Set priority of the task (1000 by default). The value must be between 900 and 1100. ' \
                       'Note that priorities of tasks are relevant only in ' \
                       "each user's share, i.e., your tasks cannot jump over other user's tasks " \
                       'even if you give higher priorities.')
optP.add_argument('--osMatching', action='store_const', const=True, dest='osMatching', default=False,
                  help='To let the brokerage choose sites which have the same OS as the local machine has.')
optP.add_argument('--cpuTimePerEvent', action='store', dest='cpuTimePerEvent', default=-1, type=int,
                help='Expected HS06 seconds per event (~= 10 * the expected duration per event in seconds)')
optP.add_argument('--maxWalltime', action='store', dest='maxWalltime', default=0, type=int,
                help='Max walltime for each job in hours. Note that this option works only ' \
                     'when the nevents metadata of input files are available in rucio')

from pandatools import MiscUtils
from pandatools.MiscUtils import commands_get_output, commands_get_status_output, commands_get_status_output_with_env

# parse options
options,args = optP.parse_known_args()
if options.verbose:
    print(options)
    print(args)
    print('')
# load json
jsonExecStr = ''
if options.loadJson is not None:
    loadOpts = MiscUtils.decodeJSON(options.loadJson)
    for k in loadOpts:
        v = loadOpts[k]
        origK = k
        if k == 'athena_args':
            args = v
            continue
        if not hasattr(options, k):
            print("ERROR: unknown parameter {0} in {1}".format(k, options.loadJson))
            sys.exit(1)
        else:
            setattr(options,k, v)
        if v is True:
            jsonExecStr += ' --{0}'.format(origK)
        else:
            if isinstance(v, types.StringType):
                jsonExecStr += " --{0}='{1}'".format(origK, v)
            else:
                jsonExecStr += " --{0}={1}".format(origK, v)
    if options.verbose:
        print("options after loading json")
        print(options)
        print('')

# display version
from pandatools import PandaToolsPkgInfo
if options.version:
    print("Version: %s" % PandaToolsPkgInfo.release_version)
    sys.exit(0)

from pandatools import Client
from pandatools import PsubUtils
from pandatools import AthenaUtils
from pandatools import PLogger

# update panda-client
if options.update:
    res = PsubUtils.updatePackage(options.verbose)
    if res:
        sys.exit(0)
    else:
        sys.exit(1)

# full execution string
fullExecString = PsubUtils.convSysArgv()
fullExecString += jsonExecStr

# get logger
tmpLog = PLogger.getPandaLogger()

# use dev server
if options.devSrv:
    Client.useDevServer()

# use INTR server
if options.intrSrv:
    Client.useIntrServer()

# noCompile uses noBuild stuff
if options.noCompile:
    if options.noBuild:
        tmpLog.error("--noBuild and --noCompile cannot be used simultaneously")
        sys.exit(EC_Config)
    options.noBuild = True

# syntax sugar
if options.nJobs > 0:
    options.split = options.nJobs

# files to be deleted
delFilesOnExit = []

# save current dir
currentDir = os.path.realpath(os.getcwd())

# exclude sites
if options.excludedSite != []:
    options.excludedSite = PsubUtils.splitCommaConcatenatedItems(options.excludedSite)

# use certain sites
includedSite = None
if re.search(',',options.site) != None:
    includedSite = PsubUtils.splitCommaConcatenatedItems([options.site])
    options.site = 'AUTO'

# site specified
siteSpecified = True
if options.site == 'AUTO':
    siteSpecified = False

# list of output files which can be skipped
options.allowNoOutput = options.allowNoOutput.split(',')

# use outputPath as outDS
if not options.outDS.endswith('/'):
    options.outDS = options.outDS + '/'

# read datasets from file
if options.inDsTxt != '':
    options.inDS = PsubUtils.readDsFromFile(options.inDsTxt)

# bulk submission
ioList = []
if options.inOutDsJson != '':
    options.bulkSubmission = True
if options.bulkSubmission:
    if options.inOutDsJson == '':
        tmpLog.error("--inOutDsJson is missing")
        sys.exit(EC_Config)
    if options.eventPickEvtList != '':
        tmpLog.error("cannnot use --eventPickEvtList and --inOutDsJson at the same time")
        sys.exit(EC_Config)
    ioList = MiscUtils.decodeJSON(options.inOutDsJson)
    for ioItem in ioList:
        if not ioItem['outDS'].endswith('/'):
            ioItem['outDS'] += '/'
    options.inDS = ioList[0]['inDS']
    options.outDS = ioList[0]['outDS']
else:
    ioList = [{'inDS': options.inDS, 'outDS': options.outDS}]

# error
if options.outDS == '':
    tmpLog.error("no outDS is given\n pathena [--inDS input] --outDS output myJobO.py")
    sys.exit(EC_Config)
if options.split < -1 :
    tmpLog.error("Number of jobs should be a positive integer")
    sys.exit(EC_Config)
if options.pfnList != '':
    if options.inDS != '': 
        tmpLog.error("--pfnList and --inDS cannot be used at the same time")
        sys.exit(EC_Config)
    if options.shipinput:
        tmpLog.error("--shipInput and --inDS cannot be used at the same time")
        sys.exit(EC_Config)
    if options.site == 'AUTO':
        tmpLog.error("--site must be specified when --pfnList is used")
        sys.exit(EC_Config)

# absolute path for PFN list
usePfnList = False
if options.pfnList != '':
    options.pfnList = os.path.realpath(options.pfnList)
    usePfnList = True

# split options are mutually exclusive
if (options.nFilesPerJob > 0 and options.nEventsPerJob > 0 and options.nGBPerJob != -1):
    tmpLog.error("split by files, split by events and split by file size can not be used simultaneously")
    sys.exit(EC_Config)

# split options are mutually exclusive
if (options.nEventsPerJob > 0 and options.nGBPerJob != -1):
    tmpLog.error("split by events and split by file size can not be used simultaneously")
    sys.exit(EC_Config)

# check nGBPerJob
if not options.nGBPerJob in [-1,'MAX']:
    # convert to int
    try:
        if options.nGBPerJob != 'MAX':
            options.nGBPerJob = int(options.nGBPerJob)
    except:
        tmpLog.error("nGBPerJob must be an integer or MAX")
        sys.exit(EC_Config)
    # check negative    
    if options.nGBPerJob <= 0:
        tmpLog.error("nGBPerJob must be positive")
        sys.exit(EC_Config)
    
# trf parameter
if options.trf == False:
    orig_trfStr = ''
else:
    orig_trfStr = options.trf

# AMI event-level split
if options.useAMIEventLevelSplit == None:
    if options.inDS.startswith('data') or options.goodRunListXML != '':
        # use AMI for real data since the number of events per file is not uniform
        options.useAMIEventLevelSplit = True
    else:
        options.useAMIEventLevelSplit = False

# check DBRelease
if options.dbRelease != '' and (options.dbRelease.find(':') == -1 and options.dbRelease !='LATEST'):
    tmpLog.error("invalid argument for --dbRelease. Must be DatasetName:FileName or LATEST")  
    sys.exit(EC_Config)

# Good Run List
if options.goodRunListXML != '' and options.inDS != '':
    tmpLog.error("cannnot use --goodRunListXML and --inDS at the same time")
    sys.exit(EC_Config)

# event picking
if options.eventPickEvtList != '' and options.inDS != '':
    tmpLog.error("cannnot use --eventPickEvtList and --inDS at the same time")
    sys.exit(EC_Config)

# param check for event picking
if options.eventPickEvtList != '':
    if options.eventPickDataType == '':
        tmpLog.error("--eventPickDataType must be specified")
        sys.exit(EC_Config)
    if options.trf != False:
        tmpLog.error("--eventPickEvtList doesn't work with --trf until official transformations support event picking")
        sys.exit(EC_Config)
        
    
# additional files
options.extFile = options.extFile.split(',')
try:
    options.extFile.remove('')
except:
    pass
options.extOutFile = re.sub(' ','',options.extOutFile)
options.extOutFile = options.extOutFile.split(',')
try:
    options.extOutFile.remove('')
except:
    pass

# user-specified merging script
if options.mergeScript != '':
    # enable merging
    options.mergeOutput = True
    # add it to extFile
    if not options.mergeScript in options.extFile:
        options.extFile.append(options.mergeScript)
    
# glue packages
options.gluePackages = options.gluePackages.split(',')
try:
    options.gluePackages.remove('')
except:
    pass

# set excludeFile
AthenaUtils.setExcludeFile(options.excludeFile)

# mapping for extra stream names
if options.appendStrToExtStream:
    AthenaUtils.enableExtendedExtStreamName()

# file list
tmpList = options.filelist.split(',')
options.filelist = []
for tmpItem in tmpList:
    if tmpItem == '':
        continue
    # wild card
    tmpItem = tmpItem.replace('*','.*')
    # append
    options.filelist.append(tmpItem) 
# read file list from file
if options.inputFileList != '':
    rFile = open(options.inputFileList)
    for line in rFile:
        line = re.sub('\n','',line)
        line = line.strip()
        if line != '':
            options.filelist.append(line)
    rFile.close()

# suppressed streams
options.supStream = options.supStream.upper().split(',')
try:
    options.supStream.remove('')
except:
    pass

# split related
if options.split > 0:
    # set nFiles when nEventsPerJob and nEventsPerFile are set
    if options.nEventsPerJob > 0 and options.nEventsPerFile > 0:
        if options.nEventsPerJob >= options.nEventsPerFile:
            options.nfiles = options.nEventsPerJob / options.nEventsPerFile * options.split
        else:
            options.nfiles =  options.split / (options.nEventsPerFile / options.nEventsPerJob)
            if options.nfiles == 0:
                options.nfiles = 1

    # set nFiles when nFilesPerJob is set
    if options.nFilesPerJob > 0 and options.nfiles == 0:
        options.nfiles = options.nFilesPerJob * options.split

    # set nFiles per job when nFiles is set
    if options.nFilesPerJob < 0 and options.nfiles > 0:
        options.nFilesPerJob = options.nfiles / options.split
        if options.nFilesPerJob == 0:
            options.nFilesPerJob = 1

# check
if options.inDS != '' and options.split > 0 and options.nFilesPerJob < 0 and options.nfiles == 0 and options.nEventsPerJob < 0:
    tmpLog.error("--split requires --nFilesPerJob or --nFiles or --nEventsPerJob when --inDS is specified")
    sys.exit(EC_Config)

# remove whitespaces
if options.inDS != '':
    options.inDS = options.inDS.replace(' ', '')

# warning
if options.nFilesPerJob > 0 and options.nFilesPerJob < 5:
    tmpLog.warning("Very small --nFilesPerJob tends to generate so many short jobs which could send your task to exhausted state "
                   "after scouts are done, since short jobs are problematic for the grid. Please consider not to use the option.")

# check grid-proxy
PsubUtils.check_proxy(options.verbose, options.vomsRoles)

# get DN
distinguishedName = PsubUtils.getDN(options.verbose)

# get nickname
nickName = PsubUtils.getNickname(options.verbose)

if nickName == '':
    sys.exit(EC_Config)

# set Rucio accounting
PsubUtils.setRucioAccount(nickName,'pathena',True)

# convert in/outTarBall to full path
if options.inTarBall != '':
    options.inTarBall = os.path.abspath(os.path.expanduser(options.inTarBall))
if options.outTarBall != '':
    options.outTarBall = os.path.abspath(os.path.expanduser(options.outTarBall))

# convert n/outRunConfig to full path
if options.inRunConfig != '':
    options.inRunConfig = os.path.abspath(os.path.expanduser(options.inRunConfig))
if options.outRunConfig != '':
    options.outRunConfig = os.path.abspath(os.path.expanduser(options.outRunConfig))

# check maxCpuCount 
if options.maxCpuCount > Client.maxCpuCountLimit:
    tmpLog.error("too large maxCpuCount. Must be less than %s" % Client.maxCpuCountLimit)
    sys.exit(EC_Config)

# create tmp dir
if options.tmpDir == '':
    tmpDir = '%s/%s' % (currentDir,MiscUtils.wrappedUuidGen())
else:
    tmpDir = '%s/%s' % (os.path.abspath(options.tmpDir),MiscUtils.wrappedUuidGen())    
os.makedirs(tmpDir)

# set tmp dir in Client
Client.setGlobalTmpDir(tmpDir)

# exit action
def _onExit(dir,files):
    for tmpFile in files:
        commands_get_output('rm -rf %s' % tmpFile)        
    commands_get_output('rm -rf %s' % dir)
atexit.register(_onExit,tmpDir,delFilesOnExit)


# get Athena versions
stA,retA = AthenaUtils.getAthenaVer()
# failed
if not stA:
    sys.exit(EC_CMT)
workArea  = retA['workArea'] 
athenaVer = retA['athenaVer'] 
groupArea = retA['groupArea'] 
cacheVer  = retA['cacheVer'] 
nightVer  = retA['nightVer']

# overwrite with athenaTag
if options.athenaTag != '':
    athenaVer = ''
    cacheVer  = ''
    nightVer  = ''
    # get list of Athena projects
    listProjects = Client.getCachePrefixes(options.verbose)
    items = options.athenaTag.split(',')
    usingNightlies = False
    for item in items:
        # releases
        match = re.search('^(\d+\.\d+\.\d+)',item)
        if match != None:
            athenaVer = match.group(1)
            # cache
            cmatch = re.search('^(\d+\.\d+\.\d+\.\d+\.*\d*)$',item)
            if cmatch != None:
                cacheVer += '_%s' % cmatch.group(1)
        else:
            # nightlies
            match = re.search('^(\d+\.\d+\.X|\d+\.X\.\d+)$',item)
            if match != None:
                athenaVer = 'Atlas-%s' % match.group(1)
        # project
        if item.startswith('Atlas') or item in listProjects:
            # ignore AtlasOffline
            if item in ['AtlasOffline']:
                continue
            cacheVer = '-'+item+cacheVer
        # nightlies    
        if item.startswith('rel_'):
            usingNightlies = True
            if 'dev' in items:
                athenaVer = 'Atlas-dev'
            elif 'devval' in items:
                athenaVer = 'Atlas-devval'
            cacheVer  = '-AtlasOffline_%s' % item
        # CMTCONFIG
        if item in ['32','64']:
            tmpLog.warning("%s in --athenaTag is unsupported. Please use --cmtConfig instead" % item)
    # check cache
    if re.search('^-.+_.+$',cacheVer) == None:
        if re.search('^_\d+\.\d+\.\d+\.\d+$',cacheVer) != None:
            # use AtlasProduction
            cacheVer = '-AtlasProduction'+cacheVer
        elif 'AthAnalysisBase' in cacheVer or 'AthAnalysis' in cacheVer:
            # AthAnalysis
            cacheVer  = cacheVer + '_%s' % athenaVer
            athenaVer = ''
        else:
            # unknown
            cacheVer = ''
    # use dev nightlies
    if usingNightlies and athenaVer == '':
        athenaVer = 'Atlas-dev'

# set CMTCONFIG
options.cmtConfig = AthenaUtils.getCmtConfig(athenaVer,cacheVer,nightVer,options.cmtConfig,verbose=options.verbose)

# check CMTCONFIG
if not AthenaUtils.checkCmtConfig(retA['cmtConfig'],options.cmtConfig,options.noBuild):
    sys.exit(EC_CMT)

tmpLog.info('using CMTCONFIG=%s' % options.cmtConfig)

# get run directory
# remove special characters                    
sString=re.sub('[\+]','.',workArea)
runDir = re.sub('^%s' % sString, '', currentDir)
if runDir == currentDir and not AthenaUtils.useCMake():
    errMsg  = "You need to run pathena in a directory under %s. " % workArea
    errMsg += "If '%s' is a read-only directory, perhaps you did setup Athena without --testarea or the 'here' tag of asetup." % workArea
    tmpLog.error(errMsg)
    sys.exit(EC_Config)
elif runDir == '':
    runDir = '.'
elif runDir.startswith('/'):
    runDir = runDir[1:]
runDir = runDir+'/'

# event picking
if options.eventPickEvtList != '':
    epLockedBy = 'pathena'
    if not options.noSubmit:
        epStat,epOutput = Client.requestEventPicking(options.eventPickEvtList,
                                                     options.eventPickDataType,
                                                     options.eventPickStreamName,
                                                     options.eventPickDS,
                                                     options.eventPickAmiTag,
                                                     options.filelist,
                                                     '',
                                                     options.outDS,
                                                     epLockedBy,
                                                     fullExecString,
                                                     1,
                                                     options.eventPickWithGUID,
                                                     options.ei_api,
                                                     options.verbose)
        # set input dataset 
        options.inDS = epOutput
    else:
        options.inDS = 'dummy'
    tmpLog.info('requested Event Picking service to stage input as %s' % options.inDS)
    # make run/event list file for event picking
    eventPickRunEvtDat = '%s/ep_%s.dat' % (currentDir,MiscUtils.wrappedUuidGen())
    evI = open(options.eventPickEvtList)
    evO = open(eventPickRunEvtDat,'w')
    evO.write(evI.read())
    # close
    evI.close()
    evO.close()
    # add to be deleted on exit
    delFilesOnExit.append(eventPickRunEvtDat)

# get job options
jobO = ''
if options.trf:
    # replace : to = for backward compatibility
    for optArg in ['DB','RNDM']:
        options.trf = re.sub('%'+optArg+':','%'+optArg+'=',options.trf)
    # use trf's parameters
    jobO = options.trf
else:
    # get jobOs from command-line
    if options.preConfig != '':
        jobO += '-p %s ' % options.preConfig
    if options.singleLine != '':
        options.singleLine = options.singleLine.replace('"','\'')
        jobO += '-c "%s" ' % options.singleLine
    for arg in args:
        jobO += ' %s' % arg
if jobO == "":
    tmpLog.error("no jobOptions is given\n   pathena [--inDS input] --outDS output myJobO.py")
    sys.exit(EC_Config)


if options.inRunConfig == '':
    # extract run configuration    
    tmpLog.info('extracting run configuration')
    # run ConfigExtractor for normal jobO 
    ret,runConfig = AthenaUtils.extractRunConfig(jobO,options.supStream,options.shipinput,
                                                 options.trf,verbose=options.verbose,
                                                 useAMI=options.useAMIAutoConf,inDS=options.inDS,
                                                 tmpDir=tmpDir)
    # save runconfig
    if options.outRunConfig != '':
        cFile = open(options.outRunConfig,'w')
        pickle.dump(runConfig,cFile)
        cFile.close()
else:
    # load from file
    ret = True
    tmpRunConfFile = open(options.inRunConfig)
    runConfig = pickle.load(tmpRunConfFile)
    tmpRunConfFile.close()
if not options.trf:
    # extractor failed
    if not ret:
        sys.exit(EC_Extractor)
    # shipped files
    if runConfig.other.inputFiles:
        for fileName in runConfig.other.inputFiles:
            # append .root for tag files
            if runConfig.other.inColl:
                match = re.search('\.root(\.\d+)*$',fileName)
                if match == None:
                    fileName = '%s.root' % fileName
            # check ship files in the current dir
            if not os.path.exists(fileName):
                tmpLog.error("%s needs exist in the current directory when --shipInput is used" % fileName)
                sys.exit(EC_Extractor)
            # append to extFile
            options.extFile.append(fileName)
            if not runConfig.input.shipFiles:
                runConfig.input['shipFiles'] = []
            if not fileName in runConfig.input['shipFiles']:    
                runConfig.input['shipFiles'].append(fileName)
    # generator files
    if runConfig.other.rndmGenFile:
        # append to extFile
        for fileName in runConfig.other.rndmGenFile:
            options.extFile.append(fileName)
    # Condition file
    if runConfig.other.condInput:
        # append to extFile
        for fileName in runConfig.other.condInput:
            if options.addPoolFC == "":
                options.addPoolFC = fileName
            else:
                options.addPoolFC += ",%s" % fileName
    # set default ref name
    if not runConfig.input.collRefName:
        runConfig.input.collRefName = 'Token'
    # check dupication in extOutFile
    if runConfig.output.alloutputs != False:
        if options.verbose:
            tmpLog.debug("output files : %s" % str(runConfig.output.alloutputs)) 
        for tmpExtOutFile in tuple(options.extOutFile):
            if tmpExtOutFile in runConfig.output.alloutputs:
                options.extOutFile.remove(tmpExtOutFile)
else:
    # parse parameters for trf
    # AMI tag
    newJobO = ''
    for tmpString in jobO.split(';'):
        match = re.search(' AMI=',tmpString)
        if match == None:
            # use original command
            newJobO += (tmpString + ';')
        else:
            tmpLog.info('getting configration from AMI')
            # get configration using GetCommand.py
            com = 'GetCommand.py ' + re.sub('^[^ ]+ ','',tmpString.strip())
            if options.verbose:
                tmpLog.debug(com)
            amiSt,amiOut = commands_get_status_output_with_env(com)
            amiSt %= 255
            if amiSt != 0:
                tmpLog.error(amiOut)
                errSt =  'Failed to get configuration from AMI. '
                errSt += 'Using AMI=tag in --trf is disallowed since it may overload the AMI server. '
                errSt += 'Please use explicit configuration parameters in --trf'
                tmpLog.error(errSt)
                sys.exit(EC_Config)
            # get full command string
            fullCommand = ''
            for amiStr in amiOut.split('\n'):
                if amiStr != '' and not amiStr.startswith('#') and not amiStr.startswith('*'):
                    fullCommand = amiStr
            # failed to extract configration        
            if fullCommand == '':
                tmpLog.error(amiOut)
                errSt =  "Failed to extract configuration from AMI's output"
                tmpLog.error(errSt)
                sys.exit(EC_Config)
            # replace
            newJobO += (fullCommand + ';')
    # remove redundant ;
    newJobO = newJobO[:-1]
    # replace
    if newJobO != '':
        jobO = newJobO
        if options.verbose:
            tmpLog.debug('new jobO : '+jobO)
    # output                
    oneOut = False
    # replace ; for job sequence
    tmpString = re.sub(';',' ',jobO)
    # look for --outputDAODFile and --reductionConf 
    match = re.search('--outputDAODFile[ =\"\']+([^ \"\',]+)',tmpString)
    outputDAODFile = None
    if match != None:
        outputDAODFile = match.group(1)
        # remove %OUT
        outputDAODFile = re.sub('%OUT\.','',outputDAODFile)
        match = re.search('--reductionConf[ =\"\']+([^ \"\']+)',tmpString)
        if match != None:
            # remove %OUT from outputDAODFile
            jobO = jobO.replace('%OUT.'+outputDAODFile,outputDAODFile)
            # loop over all configs
            reductionConf = match.group(1)
            for reductionItem in reductionConf.split(','):
                reductionItem = reductionItem.strip()
                if reductionItem == '':
                    continue
                # make actual output names for derivation
                tmpOutName = 'DAOD_{0}.{1}'.format(reductionItem,outputDAODFile)
                if not tmpOutName in options.extOutFile:
                    options.extOutFile.append(tmpOutName)
                    oneOut = True
    # look for %OUT
    for tmpItem in tmpString.split():
        match = re.search('\%OUT\.([^ \"\',]+)',tmpItem)
        if match:
            # append basenames to extOutFile
            tmpOutName = match.group(1)
            # skip basename of derivation
            if outputDAODFile != None and outputDAODFile == tmpOutName:
                continue
            if not tmpOutName in options.extOutFile:
                options.extOutFile.append(tmpOutName)
                oneOut = True
    # warning if no output
    if not oneOut:
        tmpLog.warning("argument of --trf doesn't contain any %OUT")

# no output jobs
tmpOutKeys = list(runConfig.output)
for tmpIgnorKey in ['outUserData','alloutputs']:
    try:
        tmpOutKeys.remove(tmpIgnorKey)
    except:
        pass
if tmpOutKeys == [] and options.extOutFile == [] and not options.noOutput:
    errStr  = "No output stream was extracted from jobOs or --trf. "
    if not options.trf:
        errStr += "If your job defines an output without Athena framework "
        errStr += "(e.g., using ROOT.TFile.Open instead of THistSvc) "
        errStr += "please specify the output filename by using --extOutFile. "
        errStr += "Or if you define the output with a relatively new mechanism "
        errStr += "please report it to Savannah to update the automatic extractor. "
    errStr += "If you are sure that your job doesn't produce any output file "
    errStr += "(e.g., HelloWorldOptions.py) please use --noOutput. "
    tmpLog.error(errStr)
    sys.exit(EC_Extractor)

# set extOutFile to runConfig
if options.extOutFile != []:
    runConfig.output['extOutFile'] = options.extOutFile

# check ship files in the current dir
if not runConfig.input.shipFiles:
    runConfig.input.shipFiles = []
for file in runConfig.input.shipFiles:
    if not os.path.exists(file):
        tmpLog.error("%s needs exist in the current directory when using --shipInput" % file)
        sys.exit(EC_Extractor)

# get random number
runConfig.other['rndmNumbers'] = []
if not runConfig.other.rndmStream:
    runConfig.other.rndmStream = []
if len(runConfig.other.rndmStream) != 0:
    if options.norandom:
        print()
        print("Initial random seeds need to be defined.")
        print("Enter two numbers for each random stream.")
        print("  e.g., PYTHIA : 4789899 989240512")
        print('')
    for stream in runConfig.other.rndmStream:
        if options.norandom:
            # enter manually
            while True:
                randStr = input("%s : " % stream)
                num = randStr.split()
                if len(num) == 2:
                    break
                print(" Two numbers are needed")
            runConfig.other.rndmNumbers.append([int(num[0]),int(num[1])])
        else:
            # automatic
            runConfig.other.rndmNumbers.append([random.randint(1,5000000),random.randint(1,5000000)])
    if options.norandom:
        print('')
if runConfig.other.G4RandomSeeds == True:
    if options.norandom:
        print('')
        print("Initial G4 random seeds need to be defined.")
        print("Enter one positive number.")
        print('')
        # enter manually
        while True:
            num = input("SimFlags.SeedsG4=")
            try:
                num = int(num)
                if num > 0:
                    runConfig.other.G4RandomSeeds = num
                    break
            except:
                pass
        print('')
    else:
        # automatic
        runConfig.other.G4RandomSeeds = random.randint(1,10000)
else:
    # set -1 to disable G4 Random Seeds
    runConfig.other.G4RandomSeeds = -1



#####################################################################
# input datasets

if options.inDS != '' or options.shipinput or options.pfnList != '':
    # minimum bias dataset
    if options.trf and jobO.find('%MININ') != -1:
        runConfig.input.inMinBias = True
    if runConfig.input.inMinBias:
        options.minDS,options.nMin = MiscUtils.getDatasetNameAndNumFiles(options.minDS,
                                                                         options.nMin,
                                                                         'Minimum-Bias')
    # low pT minimum bias dataset
    if options.trf and jobO.find('%LOMBIN') != -1:
        runConfig.input.inLoMinBias = True
    if runConfig.input.inLoMinBias:
        options.lowMinDS,options.nLowMin = MiscUtils.getDatasetNameAndNumFiles(options.lowMinDS,
                                                                               options.nLowMin,
                                                                               'Low pT Minimum-Bias')
    # high pT minimum bias dataset
    if options.trf and jobO.find('%HIMBIN') != -1:
        runConfig.input.inHiMinBias = True
    if runConfig.input.inHiMinBias:
        options.highMinDS,options.nHighMin = MiscUtils.getDatasetNameAndNumFiles(options.highMinDS,
                                                                                 options.nHighMin,
                                                                                 'High pT Minimum-Bias')
    # cavern dataset
    if options.trf and jobO.find('%CAVIN') != -1:
        runConfig.input.inCavern = True
    if runConfig.input.inCavern:
        options.cavDS,options.nCav = MiscUtils.getDatasetNameAndNumFiles(options.cavDS,
                                                                         options.nCav,
                                                                         'Cavern')
    # beam halo dataset
    if options.trf and jobO.find('%BHIN') != -1:
        runConfig.input.inBeamHalo = True 
    if runConfig.input.inBeamHalo:
        # use common DS
        if options.useCommonHalo:
            options.beamHaloDS,options.nBeamHalo = MiscUtils.getDatasetNameAndNumFiles(options.beamHaloDS,
                                                                                       options.nBeamHalo,
                                                                                       'BeamHalo')
        else:
            # get DS for A-side
            options.beamHaloADS,options.nBeamHaloA = MiscUtils.getDatasetNameAndNumFiles(options.beamHaloADS,
                                                                                         options.nBeamHaloA,
                                                                                         'BeamHalo A-side')
            # get DS for C-side
            options.beamHaloCDS,options.nBeamHaloC = MiscUtils.getDatasetNameAndNumFiles(options.beamHaloCDS,
                                                                                         options.nBeamHaloC,
                                                                                         'BeamHalo C-side')
    # beam gas dataset
    if options.trf and jobO.find('%BGIN') != -1:
        runConfig.input.inBeamGas = True  
    if runConfig.input.inBeamGas: 
        # use common DS
        if options.useCommonGas:
            options.beamGasDS,options.nBeamGas = MiscUtils.getDatasetNameAndNumFiles(options.beamGasDS,
                                                                                     options.nBeamGas,
                                                                                     'BeamGas')
        else:
            # get DS for H
            options.beamGasHDS,options.nBeamGasH = MiscUtils.getDatasetNameAndNumFiles(options.beamGasHDS,
                                                                                       options.nBeamGasH,
                                                                                       'BeamGas Hydrogen')
            # get DS for C
            options.beamGasCDS,options.nBeamGasC = MiscUtils.getDatasetNameAndNumFiles(options.beamGasCDS,
                                                                                       options.nBeamGasC,
                                                                                       'BeamGas Carbon')
            # get DS for O
            options.beamGasODS,options.nBeamGasO = MiscUtils.getDatasetNameAndNumFiles(options.beamGasODS,
                                                                                       options.nBeamGasO,
                                                                                       'BeamGas Oxygen')



#####################################################################
# archive sources and send it to HTTP-reachable location

if True:
    if options.inTarBall == '':
        # extract jobOs with full pathnames
        for tmpItem in jobO.split():
            if re.search('^/.*\.py$',tmpItem) != None:
                # set random name to avoid overwriting
                tmpName = tmpItem.split('/')[-1]
                tmpName = '%s_%s' % (MiscUtils.wrappedUuidGen(),tmpName)
                # set
                AthenaUtils.fullPathJobOs[tmpItem] = tmpName

        # copy some athena specific files
        AthenaUtils.copyAthenaStuff(currentDir)

        # set extFile
        AthenaUtils.setExtFile(options.extFile)

        archiveName = ""
        if not (options.noBuild and not options.noCompile):
                # archive with cpack
            if AthenaUtils.useCMake():
                archiveName,archiveFullName = AthenaUtils.archiveWithCpack(True,tmpDir,options.verbose)
            # archive sources
            archiveName,archiveFullName = AthenaUtils.archiveSourceFiles(workArea,runDir,currentDir,tmpDir,
                                                                         options.verbose,options.gluePackages,
                                                                         archiveName=archiveName) 
        else:
                # archive with cpack
            if AthenaUtils.useCMake():
                archiveName,archiveFullName = AthenaUtils.archiveWithCpack(False,tmpDir,options.verbose)
            # archive jobO
            archiveName,archiveFullName = AthenaUtils.archiveJobOFiles(workArea,runDir,currentDir,
                                                                       tmpDir,options.verbose,
                                                                       archiveName=archiveName)
            
        # archive InstallArea
        AthenaUtils.archiveInstallArea(workArea,groupArea,archiveName,archiveFullName,
                                       tmpDir,options.noBuild,options.verbose)
        # back to tmp dir        
        os.chdir(tmpDir)
        # remove some athena specific files
        AthenaUtils.deleteAthenaStuff(currentDir)
        # compress
        status,out = commands_get_status_output('gzip -f %s' % archiveName)
        if status != 0 or options.verbose:
            print(out)
        archiveName += '.gz'
        # check archive
        status,out = commands_get_status_output('ls -l %s' % archiveName)
        if status != 0:
            print(out)
            tmpLog.error("Failed to archive working area.\n        If you see 'Disk quota exceeded', try '--tmpDir /tmp'") 
            sys.exit(EC_Archive)

        # check symlinks
        tmpLog.info("checking symbolic links")
        status,out = commands_get_status_output('tar tvfz %s' % archiveName)
        if status != 0:
            tmpLog.error("Failed to expand archive")
            sys.exit(EC_Archive)
        symlinks = []    
        for line in out.split('\n'):
            items = line.split()
            if items[0].startswith('l') and items[-1].startswith('/'):
                symlinks.append(line)
        if symlinks != []:
            tmpStr  = "Found some unresolved symlinks which may cause a problem\n"
            tmpStr += "     See, e.g., http://savannah.cern.ch/bugs/?43885\n"
            tmpStr += "   Please ignore if you believe they are harmless"
            tmpLog.warning(tmpStr)
            for symlink in symlinks:
                print("  %s" % symlink)
    else:
        # go to tmp dir
        os.chdir(tmpDir)
        # use a saved copy
        if not (options.noBuild and not options.noCompile):
            archiveName     = 'sources.%s.tar' % MiscUtils.wrappedUuidGen()
            archiveFullName = "%s/%s" % (tmpDir,archiveName)
        else:
            archiveName     = 'jobO.%s.tar' % MiscUtils.wrappedUuidGen()
            archiveFullName = "%s/%s" % (tmpDir,archiveName)
        # make copy to avoid name duplication
        shutil.copy(options.inTarBall,archiveFullName)
        
    # save
    if options.outTarBall != '':
        shutil.copy(archiveName,options.outTarBall)

    # put sources/jobO via HTTP POST
    if not options.noSubmit:
        tmpLog.info("uploading source/jobO files")
        status,out = Client.putFile(archiveName,options.verbose,useCacheSrv=True,reuseSandbox=True)
        if out.startswith('NewFileName:'):
            # found the same input sandbox to reuse
            archiveName = out.split(':')[-1]
        elif out != 'True':
            # failed
            print(out)
            tmpLog.error("Failed with %s" % status)
            sys.exit(EC_Post)
        # good run list
        if options.goodRunListXML != '':
            options.goodRunListXML = PsubUtils.uploadGzippedFile(options.goodRunListXML,currentDir,tmpLog,delFilesOnExit,
                                                                 options.noSubmit,options.verbose)




# special handling
specialHandling = ''
if options.express:
    specialHandling += 'express,'
if options.debugMode:
    specialHandling += 'debug,'
specialHandling = specialHandling[:-1]


####################################################################3
# submit jobs

# append tmpdir to import taskbuffer module
sys.path = [tmpDir]+sys.path

# make task
taskParamMap = {}
taskParamMap['taskName'] = options.outDS
if not options.allowTaskDuplication:
    taskParamMap['uniqueTaskName'] = True
taskParamMap['vo'] = 'atlas'
taskParamMap['architecture'] = AthenaUtils.getCmtConfigImg(athenaVer,cacheVer,nightVer,options.cmtConfig)
if athenaVer != '':
    taskParamMap['transUses'] = 'Atlas-%s' % athenaVer
else:
    taskParamMap['transUses'] = athenaVer
taskParamMap['transHome'] = 'AnalysisTransforms'+cacheVer+nightVer
taskParamMap['processingType'] = 'panda-client-{0}-jedi-athena'.format(PandaToolsPkgInfo.release_version)
if options.trf:
    taskParamMap['processingType'] += '-trf'
if options.eventPickEvtList != '':
    taskParamMap['processingType'] += '-evp'
    taskParamMap['waitInput'] = 1
if options.goodRunListXML != '':
    taskParamMap['processingType'] += '-grl'
if options.prodSourceLabel == '':
    taskParamMap['prodSourceLabel'] = 'user'
else:
    taskParamMap['prodSourceLabel'] = options.prodSourceLabel
if options.site != 'AUTO':
    taskParamMap['site'] = options.site
else:
    taskParamMap['site'] = None
taskParamMap['excludedSite'] = options.excludedSite
if includedSite != None and includedSite != []:
    taskParamMap['includedSite'] = includedSite
else:
    taskParamMap['includedSite'] = None
if options.priority is not None:
    taskParamMap['currentPriority'] = options.priority
if options.nfiles > 0:
    taskParamMap['nFiles'] = options.nfiles
if options.nFilesPerJob > 0:
    taskParamMap['nFilesPerJob'] = options.nFilesPerJob
if not options.nGBPerJob in [-1,'MAX']:
    # don't set MAX since it is the defalt on the server side
    taskParamMap['nGBPerJob'] = options.nGBPerJob
if options.nEventsPerJob > 0:
    taskParamMap['nEventsPerJob'] = options.nEventsPerJob
    if options.nEventsPerFile <= 0:
        taskParamMap['useRealNumEvents'] = True
if options.nEventsPerFile > 0:
    taskParamMap['nEventsPerFile'] = options.nEventsPerFile
if options.split > 0 and options.nEventsPerJob > 0:
    taskParamMap['nEvents'] = options.split*options.nEventsPerJob
taskParamMap['cliParams'] = fullExecString
if options.noEmail:
    taskParamMap['noEmail'] = True
if options.skipScout:
    taskParamMap['skipScout'] = True
if options.respectSplitRule:
    taskParamMap['respectSplitRule'] = True
if options.disableAutoRetry:
    taskParamMap['disableAutoRetry'] = 1
if options.workingGroup != None:
    taskParamMap['workingGroup'] = options.workingGroup
if options.official:
    taskParamMap['official'] = True
if options.useNewCode:
    taskParamMap['fixedSandbox'] = archiveName
if options.maxCpuCount > 0:
    taskParamMap['walltime'] = -options.maxCpuCount
if options.maxWalltime > 0:
    taskParamMap['maxWalltime'] = options.maxWalltime
if options.cpuTimePerEvent > 0:
    taskParamMap['cpuTime'] = options.cpuTimePerEvent
    taskParamMap['cpuTimeUnit'] = 'HS06sPerEvent'
if options.memory > 0:
    taskParamMap['ramCount'] = options.memory
if options.nCore > 1:
    taskParamMap['coreCount'] = options.nCore
elif options.nThreads > 1:
    taskParamMap['coreCount'] = options.nThreads
if options.skipFilesUsedBy != '':
    taskParamMap['skipFilesUsedBy'] = options.skipFilesUsedBy
taskParamMap['respectSplitRule'] = True
if options.maxAttempt > 0 and options.maxAttempt <= 50:
    taskParamMap['maxAttempt'] = options.maxAttempt
if options.osMatching:
    taskParamMap['osMatching'] = True
taskParamMap['osInfo'] = PsubUtils.get_os_information()
# source URL
matchURL = re.search("(http.*://[^/]+)/",Client.baseURLCSRVSSL)
if matchURL != None:
    taskParamMap['sourceURL'] = matchURL.group(1)
# middle name
if options.addNthFieldOfInFileToLFN != '':
    taskParamMap['addNthFieldToLFN'] = options.addNthFieldOfInFileToLFN
    taskParamMap['useFileAsSourceLFN'] = True
elif options.addNthFieldOfInDSToLFN != '':
    taskParamMap['addNthFieldToLFN'] = options.addNthFieldOfInDSToLFN
# dataset name
logDatasetName = re.sub('/$','.log/',options.outDS)
# log
taskParamMap['log'] = {'dataset': logDatasetName,
                       'container': logDatasetName,
                       'type':'template',
                       'param_type':'log',
                       'value':'{0}.$JEDITASKID.${{SN}}.log.tgz'.format(logDatasetName[:-1])
                       }
if options.addNthFieldOfInFileToLFN != '':
    loglfn  = '{0}.{1}'.format(*logDatasetName.split('.')[:2])
    loglfn += '${MIDDLENAME}.$JEDITASKID._${SN}.log.tgz'
    taskParamMap['log']['value'] = loglfn
if options.spaceToken != '':
    taskParamMap['log']['token'] = options.spaceToken

# make job parameters
taskParamMap['jobParameters'] = []

# build
if options.noBuild and not options.noCompile:
    taskParamMap['jobParameters'] += [
        {'type':'constant',
         'value': '-a {0}'.format(archiveName),
         },
        ]
else:
    taskParamMap['jobParameters'] += [
        {'type':'constant',
         'value': '-l ${LIB}',
         },
        ]
# pre execution string
pStr1 = ''    
if runConfig.other.rndmStream != []:
    pStr1 = "AtRndmGenSvc=Service('AtRndmGenSvc');AtRndmGenSvc.Seeds=["
    for stream in runConfig.other.rndmStream:
        num = runConfig.other.rndmNumbers[runConfig.other.rndmStream.index(stream)]
        pStr1 += "'%s ${RNDMSEED} %s'," % (stream,num[1])
    pStr1 += "]"
    dictItem = {'type':'template',
                'param_type':'number',
                'value':'${RNDMSEED}',
                'hidden':True,
                'offset':runConfig.other.rndmStream[0][0],
                }
    taskParamMap['jobParameters'] += [dictItem]
# split by event option was invoked
pStr2 = ''
if options.nEventsPerJob > 0 and (not options.trf):
    # @ Number of events to be processed per job
    param1 = "theApp.EvtMax=${MAXEVENTS}"
    # @ possibly skip events in a file
    if runConfig.input.noInput:
        pStr2 = param1
    else:
        param2 = "EventSelector.SkipEvents=${SKIPEVENTS}"
        # @ Form a string to add to job parameters
        pStr2 = '%s;%s' % (param1,param2)
# set pre execution parameter
if pStr1 != '' or pStr2 != '':
    if pStr1 == '' or pStr2 == '':
        preStr = pStr1+pStr2
    else:
        preStr = "%s;%s" % (pStr1,pStr2)
    taskParamMap['jobParameters'] += [
        {'type':'constant',
         'value': '-f "',
         'padding':False,
         },
        ]
    taskParamMap['jobParameters'] += PsubUtils.convertParamStrToJediParam(preStr,{},'',
                                                                          False,False)
    taskParamMap['jobParameters'] += [
        {'type':'constant',
         'value': '"',
         },
        ]

# misc
param  = '--sourceURL ${SURL} '
param += '-r {0} '.format(runDir)
# addPoolFC
if options.addPoolFC != "":
    param += '--addPoolFC %s ' % options.addPoolFC
# disable to skip missing files
if options.notSkipMissing:
    param += '--notSkipMissing '
# given PFN 
if options.pfnList != '':
    param += '--givenPFN '
# run TRF
if options.trf:
    param += '--trf '
# general input format
if options.generalInput:
    param += '--generalInput '
# use local access for TRF and BS
if (options.trf or runConfig.input.inBS) and not options.forceDirectIO:
    param += '--useLocalIO '        
# use theApp.nextEvent
if options.useNextEvent:
    param += '--useNextEvent '
# use CMake
if AthenaUtils.useCMake():
    param += "--useCMake "
# AthenaMT
if options.nThreads > 1:
    param += "--useAthenaMT "
# use code tracer 
if options.codeTrace:
    param += '--codeTrace '
# debug parameters
if options.queueData != '':
    param += "--overwriteQueuedata=%s " % options.queueData
# read BS    
if runConfig.input.inBS:
    param += '-b '
# use back navigation
if runConfig.input.backNavi:
    param += '-e '
# ship input
if options.shipinput:
    param += '--shipInput '
# event picking
if options.eventPickEvtList != '':
    param += '--eventPickTxt=%s ' % eventPickRunEvtDat.split('/')[-1]
# container
if options.containerImage != '':
    param += "--containerImage {0} ".format(options.containerImage)
# assign
if param != '':
    taskParamMap['jobParameters'] += [
        {'type':'constant',
         'value': param,
         },
        ]

# input
inputMap = {}
if options.inDS != '':
    tmpDict = {'type':'template',
               'param_type':'input',
               'value':'-i "${IN/T}"',
               'dataset':options.inDS,
               'expand':True,
               'exclude':'\.log\.tgz(\.\d+)*$',
               }
    if options.inputType != '':
        tmpDict['include'] = options.inputType
    if options.filelist != []:
        tmpDict['files'] = options.filelist
    taskParamMap['jobParameters'].append(tmpDict)
    taskParamMap['dsForIN'] = options.inDS
    inputMap['IN'] = options.inDS
elif options.pfnList != '':
    taskParamMap['pfnList'] = PsubUtils.getListPFN(options.pfnList)
    # use noInput mecahism
    taskParamMap['noInput'] = True
    if options.nfiles == 0:
        taskParamMap['nFiles'] = len(taskParamMap['pfnList'])
    taskParamMap['jobParameters'] += [
        {'type':'constant',
         'value':'-i "${IN/T}"',
         },
        ]
elif options.goodRunListXML != '':
    tmpDict = {'type':'template',
               'param_type':'input',
               'value':'-i "${IN/T}"',
               'dataset':'%%INDS%%',
               'expand':True,
               'exclude':'\.log\.tgz(\.\d+)*$',
               'files':'%%INLFNLIST%%',
               }
    taskParamMap['jobParameters'].append(tmpDict)
    taskParamMap['dsForIN'] = '%%INDS%%'
    inputMap['IN'] ='%%INDS%%'
else:
    # no input
    taskParamMap['noInput'] = True
    if options.nEventsPerJob > 0:
        taskParamMap['nEventsPerJob'] = options.nEventsPerJob
    else:
        taskParamMap['nEventsPerJob'] = 1
    if options.split > 0:
        taskParamMap['nEvents'] = options.split
    else:
        taskParamMap['nEvents'] = 1
    taskParamMap['nEvents'] *= taskParamMap['nEventsPerJob']
    taskParamMap['jobParameters'] += [
        {'type':'constant',
         'value': '-i "[]"',
         },
        ]

# extract DBR for --trf
dbrInTRF = False
if options.trf:
    tmpMatch = re.search('%DB=([^ \'\";]+)',jobO)
    if tmpMatch != None:
        options.dbRelease = tmpMatch.group(1)
        dbrInTRF = True
# param for DBR     
if options.dbRelease != '':
    dbrDS = options.dbRelease.split(':')[0]
    # change LATEST to DBR_LATEST
    if dbrDS == 'LATEST':
        dbrDS = 'DBR_LATEST'
    dictItem = {'type':'template',
                'param_type':'input',
                'value':'--dbrFile=${DBR}',
                'dataset':dbrDS,
                }
    taskParamMap['jobParameters'] += [dictItem]
    # no expansion
    if dbrInTRF:
        dictItem = {'type':'constant',
                    'value':'--noExpandDBR',
                    }
        taskParamMap['jobParameters'] += [dictItem]

# minimum bias
minBiasStream = ''        
if options.minDS != '':
    dictItem = MiscUtils.makeJediJobParam('${MININ}',options.minDS,'input',hidden=True,
                                          expand=True,exclude='\.log\.tgz(\.\d+)*$',
                                          nFilesPerJob=options.nMin,useNumFilesAsRatio=True,
                                          randomAtt=options.randomMin,reusableAtt=options.sameSecRetry)
    taskParamMap['jobParameters'] += dictItem
    inputMap['MININ'] = options.minDS
    minBiasStream += 'MININ,'
if options.lowMinDS != '':
    dictItem = MiscUtils.makeJediJobParam('${LOMBIN}',options.lowMinDS,'input',hidden=True,
                                          expand=True,exclude='\.log\.tgz(\.\d+)*$',
                                          nFilesPerJob=options.nLowMin,useNumFilesAsRatio=True,
                                          randomAtt=options.randomMin,reusableAtt=options.sameSecRetry)
    taskParamMap['jobParameters'] += dictItem
    inputMap['LOMBIN'] = options.lowMinDS
    minBiasStream += 'LOMBIN,'
if options.highMinDS != '':
    dictItem = MiscUtils.makeJediJobParam('${HIMBIN}',options.highMinDS,'input',hidden=True,
                                          expand=True,exclude='\.log\.tgz(\.\d+)*$',
                                          nFilesPerJob=options.nHighMin,useNumFilesAsRatio=True,
                                          randomAtt=options.randomMin,reusableAtt=options.sameSecRetry)
    taskParamMap['jobParameters'] += dictItem
    inputMap['HIMBIN'] = options.highMinDS
    minBiasStream += 'HIMBIN,'
minBiasStream = minBiasStream[:-1]
if minBiasStream != '':
    dictItem = {'type':'constant',
                'value':'-m "${{{0}/T}}"'.format(minBiasStream),
                }
    taskParamMap['jobParameters'] += [dictItem]
    
    
# cavern
if options.cavDS != '':
    dictItem = MiscUtils.makeJediJobParam('-n "${CAVIN/T}"',options.cavDS,'input',
                                          expand=True,exclude='\.log\.tgz(\.\d+)*$',
                                          nFilesPerJob=options.nCav,useNumFilesAsRatio=True,
                                          randomAtt=options.randomCav,reusableAtt=options.sameSecRetry)
    taskParamMap['jobParameters'] += dictItem
    inputMap['CAVIN'] = options.cavDS


# beam halo
beamHaloStream = ''
if options.beamHaloDS != '':
    dictItem = MiscUtils.makeJediJobParam('${BHIN}',options.beamHaloDS,'input',hidden=True,
                                          expand=True,exclude='\.log\.tgz(\.\d+)*$',
                                          nFilesPerJob=options.nBeamHalo,useNumFilesAsRatio=True,
                                          reusableAtt=options.sameSecRetry)
    taskParamMap['jobParameters'] += dictItem
    inputMap['BHIN'] = options.beamHaloDS
    beamHaloStream += 'BHIN,'    
if options.beamHaloADS != '':
    dictItem = MiscUtils.makeJediJobParam('${BHAIN}',options.beamHaloADS,'input',hidden=True,
                                          expand=True,exclude='\.log\.tgz(\.\d+)*$',
                                          nFilesPerJob=options.nBeamHaloA,useNumFilesAsRatio=True,
                                          reusableAtt=options.sameSecRetry)
    taskParamMap['jobParameters'] += dictItem
    inputMap['BHAIN'] = options.beamHaloADS
    beamHaloStream += 'BHAIN,'
if options.beamHaloCDS != '':
    dictItem = MiscUtils.makeJediJobParam('${BHCIN}',options.beamHaloCDS,'input',hidden=True,
                                          expand=True,exclude='\.log\.tgz(\.\d+)*$',
                                          nFilesPerJob=options.nBeamHaloC,useNumFilesAsRatio=True,
                                          reusableAtt=options.sameSecRetry)
    taskParamMap['jobParameters'] += dictItem
    inputMap['BHCIN'] = options.beamHaloCDS
    beamHaloStream += 'BHCIN,'    
beamHaloStream = beamHaloStream[:-1]
if beamHaloStream != '':
    dictItem = {'type':'constant',
                'value':'--beamHalo "${{{0}/T}}"'.format(beamHaloStream)
                }
    taskParamMap['jobParameters'] += [dictItem]


# beam gas
beamGasStream = ''
if options.beamGasDS != '':
    dictItem = MiscUtils.makeJediJobParam('${BGIN}',options.beamGasDS,'input',hidden=True,
                                          expand=True,exclude='\.log\.tgz(\.\d+)*$',
                                          nFilesPerJob=options.nBeamGas,useNumFilesAsRatio=True,
                                          reusableAtt=options.sameSecRetry)
    taskParamMap['jobParameters'] += dictItem
    inputMap['BGIN'] = options.beamGasDS
    beamGasStream += 'BGIN,'    
if options.beamGasHDS != '':
    dictItem = MiscUtils.makeJediJobParam('${BGHIN}',options.beamGasHDS,'input',hidden=True,
                                          expand=True,exclude='\.log\.tgz(\.\d+)*$',
                                          nFilesPerJob=options.nBeamGasH,useNumFilesAsRatio=True,
                                          reusableAtt=options.sameSecRetry)
    taskParamMap['jobParameters'] += dictItem
    inputMap['BGHIN'] = options.beamGasHDS
    beamGasStream += 'BGHIN,'    
if options.beamGasCDS != '':
    dictItem = MiscUtils.makeJediJobParam('${BGCIN}',options.beamGasCDS,'input',hidden=True,
                                          expand=True,exclude='\.log\.tgz(\.\d+)*$',
                                          nFilesPerJob=options.nBeamGasC,useNumFilesAsRatio=True,
                                          reusableAtt=options.sameSecRetry)
    taskParamMap['jobParameters'] += dictItem
    inputMap['BGCIN'] = options.beamGasHDS
    beamGasStream += 'BGCIN,'    
if options.beamGasODS != '':
    dictItem = MiscUtils.makeJediJobParam('${BGOIN}',options.beamGasODS,'input',hidden=True,
                                          expand=True,exclude='\.log\.tgz(\.\d+)*$',
                                          nFilesPerJob=options.nBeamGasO,useNumFilesAsRatio=True,
                                          reusableAtt=options.sameSecRetry)
    taskParamMap['jobParameters'] += dictItem
    inputMap['BGOIN'] = options.beamGasODS
    beamGasStream += 'BGOIN,'    
beamGasStream = beamGasStream[:-1]
if beamGasStream != '':
    dictItem = {'type':'constant',
                'value':'--beamGas "${{{0}/T}}"'.format(beamGasStream)
                }
    taskParamMap['jobParameters'] += [dictItem]


# output 
if options.addNthFieldOfInDSToLFN != '' or options.addNthFieldOfInFileToLFN != '':
    descriptionInLFN = '${MIDDLENAME}'
else:
    descriptionInLFN = ''
outMap,tmpParamList = AthenaUtils.convertConfToOutput(runConfig,options.extOutFile,options.outDS,
                                                      destination=options.destSE,spaceToken=options.spaceToken,
                                                      descriptionInLFN=descriptionInLFN,
                                                      allowNoOutput=options.allowNoOutput)
taskParamMap['jobParameters'] += [
    {'type':'constant',
     'value': '-o "%s" ' % outMap
     },
    ]
taskParamMap['jobParameters'] += tmpParamList 


# jobO parameter
if not options.trf:
    tmpJobO = jobO
    # replace full-path jobOs
    for tmpFullName in AthenaUtils.fullPathJobOs:
        tmpLocalName = AthenaUtils.fullPathJobOs[tmpFullName]
        tmpJobO = re.sub(tmpFullName,tmpLocalName,tmpJobO)
    # modify one-liner for G4 random seeds
    if runConfig.other.G4RandomSeeds > 0:
        if options.singleLine != '':
            tmpJobO = re.sub('-c "%s" ' % options.singleLine,
                             '-c "%s;from G4AtlasApps.SimFlags import SimFlags;SimFlags.SeedsG4=${RNDMSEED}" ' \
                                 % options.singleLine,tmpJobO)
        else:
            tmpJobO = '-c "from G4AtlasApps.SimFlags import SimFlags;SimFlags.SeedsG4=${RNDMSEED}" ' + tmpJobO
        dictItem = {'type':'template',
                    'param_type':'number',
                    'value':'${RNDMSEED}',
                    'hidden':True,
                    'offset':runConfig.other.G4RandomSeeds,
                    }
        taskParamMap['jobParameters'] += [dictItem]
else:
    # replace parameters for TRF
    tmpJobO = jobO
    # output : basenames are in outMap['IROOT'] trough extOutFile
    tmpOutMap = []
    for tmpName,tmpLFN in outMap['IROOT']:
        tmpJobO = tmpJobO.replace('%OUT.' + tmpName,tmpName)
    # replace DBR
    tmpJobO = re.sub('%DB=[^ \'\";]+','${DBR}',tmpJobO)
# set jobO parameter
taskParamMap['jobParameters'] += [
    {'type':'constant',
     'value': '-j "',
     'padding':False,
     },
    ]
taskParamMap['jobParameters'] += PsubUtils.convertParamStrToJediParam(tmpJobO,inputMap,options.outDS[:-1],
                                                                      True,False,usePfnList)
taskParamMap['jobParameters'] += [
    {'type':'constant',
     'value': '"',
     },
    ]

# use local IO for trf
if (options.trf or runConfig.input.inBS) and not options.forceDirectIO:
    taskParamMap['useLocalIO'] = 1

# use AMI to get the number of events per file
if options.useAMIEventLevelSplit == True:
    taskParamMap['getNumEventsInMetadata'] = True

# force stage-in
if options.forceStaged:
    taskParamMap['useLocalIO'] = 1

# build step
if options.noBuild and not options.noCompile:
    pass
else:
    jobParameters = '-i ${IN} -o ${OUT} --sourceURL ${SURL} '
    # no compile
    if options.noCompile:
        jobParameters += "--noCompile "
    # use CMake
    if AthenaUtils.useCMake():
        jobParameters += "--useCMake "
    # debug parameters
    if options.queueData != '':
        jobParameters += "--overwriteQueuedata=%s " % options.queueData
    # container
    if options.containerImage != '':
        jobParameters += "--containerImage {0} ".format(options.containerImage)
    # set task param
    taskParamMap['buildSpec'] = {
        'prodSourceLabel':'panda',
        'archiveName':archiveName,
        'jobParameters':jobParameters,
        }
    if options.prodSourceLabel != '':
         taskParamMap['buildSpec']['prodSourceLabel'] = options.prodSourceLabel

# preprocessing step

# good run list
if options.goodRunListXML != '':
    jobParameters = "--goodRunListXML {0} ".format(options.goodRunListXML)
    if options.goodRunDataType != '':
        jobParameters += "--goodRunListDataType {0} ".format(options.goodRunDataType)
    if options.goodRunProdStep != '':
        jobParameters += "--goodRunListProdStep {0} ".format(options.goodRunProdStep)
    if options.goodRunListDS != '':
        jobParameters += "--goodRunListDS {0} ".format(options.goodRunListDS)
    jobParameters += "--sourceURL ${SURL} "
    # set task param
    taskParamMap['preproSpec'] = {
        'prodSourceLabel':'panda',
        'jobParameters':jobParameters,
        }
    if options.prodSourceLabel != '':
         taskParamMap['preproSpec']['prodSourceLabel'] = options.prodSourceLabel

# merging
if options.mergeOutput:
    jobParameters = '-r {0} '.format(runDir)
    if options.mergeScript != '':
        jobParameters += '-j "{0}" '.format(options.mergeScript)
    if not options.noBuild:
        jobParameters += '-l ${LIB} '
    else:
        jobParameters += '-a {0} '.format(archiveName)
        jobParameters += "--sourceURL ${SURL} "
    jobParameters += "--useAthenaPackages "
    if AthenaUtils.useCMake():
        jobParameters += "--useCMake "
    if options.containerImage != '':
        jobParameters += "--containerImage {0} ".format(options.containerImage)
    jobParameters += '${TRN_OUTPUT:OUTPUT} ${TRN_LOG:LOG}'
    taskParamMap['mergeSpec'] = {}
    taskParamMap['mergeSpec']['useLocalIO'] = 1
    taskParamMap['mergeSpec']['jobParameters'] = jobParameters
    taskParamMap['mergeOutput'] = True
    if options.nGBPerMergeJob > 0:
        taskParamMap['nGBPerMergeJob'] = options.nGBPerMergeJob



#####################################################################
# submission

exitCode = 0
dumpList = []

# submit task
for iSubmission, ioItem in enumerate(ioList):
    if options.verbose:
        print("== parameters ==")
        print("Site       : %s" % options.site)
        print("Athena     : %s" % athenaVer)
        if groupArea != '':
            print("Group Area : %s" % groupArea)
        if cacheVer != '':
            print("ProdCache  : %s" % cacheVer[1:])
        if nightVer != '':
            print("Nightly    : %s" % nightVer[1:])
        print("cmtConfig  : %s" % AthenaUtils.getCmtConfigImg(athenaVer,cacheVer,nightVer,options.cmtConfig))
        print("RunDir     : %s" % runDir)
        print("jobO       : %s" % jobO.lstrip())

    if len(ioList) == 1:
        newTaskParamMap = taskParamMap
    else:
        # replace input and output
        options.inDS = ioItem['inDS']
        options.outDS = ioItem['outDS']
        newTaskParamMap = PsubUtils.replaceInputOutput(taskParamMap, ioItem['inDS'],
                                                       ioItem['outDS'], iSubmission)
    taskID = None
    # check outDS format
    if not PsubUtils.checkOutDsName(options.outDS,distinguishedName,options.official,nickName,
                                    options.mergeOutput, options.verbose):
        tmpLog.error("invalid output datasetname:%s" % options.outDS)
        sys.exit(EC_Config)
    # check task parameters
    exitCode, tmpStr = PsubUtils.checkTaskParam(newTaskParamMap, options.unlimitNumOutputs)
    if exitCode != 0 and len(ioList) == 1:
        sys.exit(exitCode)
    if options.noSubmit:
        if options.verbose:
            tmpLog.debug("==== taskParams ====")
            tmpKeys = list(newTaskParamMap)
            tmpKeys.sort()
            for tmpKey in tmpKeys:
                print('%s : %s' % (tmpKey, newTaskParamMap[tmpKey]))
    if not options.noSubmit and exitCode == 0:
        tmpLog.info("submit {0}".format(options.outDS))
        status,tmpOut = Client.insertTaskParams(newTaskParamMap, options.verbose, True)
        # result
        if status != 0:
            tmpStr = "task submission failed with {0}".format(status)
            tmpLog.error(tmpStr)
            exitCode = EC_Submit
        if tmpOut[0] in [0,3]:
            tmpStr = tmpOut[1]
            tmpLog.info(tmpStr)
            try:
                m = re.search('jediTaskID=(\d+)', tmpStr)
                taskID = int(m.group(1))
            except Exception:
                pass
        else:
            tmpStr = "task submission failed. {0}".format(tmpOut[1])
            tmpLog.error(tmpStr)
            exitCode = EC_Submit 
    dumpItem = copy.deepcopy(vars(options))
    dumpItem['returnCode'] = exitCode
    dumpItem['returnOut'] = tmpStr
    dumpItem['jediTaskID'] = taskID
    if len(ioList) > 1:
        dumpItem['bulkSeqNumber'] = iSubmission
    dumpList.append(dumpItem)

# go back to current dir
os.chdir(currentDir)
# dump
if options.dumpJson is not None:
    with open(options.dumpJson, 'w') as f:
        json.dump(dumpList, f)
# succeeded
sys.exit(0)
