#!/Users/boydb1/anaconda/bin/python
# -*- coding: utf-8 -*-

'''
Created on Mar 13, 2013

@author: Benjamin Yvernault, Electrical Engineering, Vanderbilt University
'''

import os,sys
from datetime import datetime
import subprocess
from subprocess import CalledProcessError
from email.mime.text import MIMEText
from email.MIMEBase import MIMEBase
import smtplib
from email import Encoders

from pyxnat import Interface

from dax import RESULTS_DIR
from dax.task import READY_TO_COMPLETE, COMPLETE, UPLOADING, JOB_FAILED, JOB_PENDING

_READY_FLAG_FILE = 'READY_TO_UPLOAD.txt'
_FAILED_FLAG_FILE = 'JOB_FAILED.txt'
_EMAILED_FLAG_FILE = 'ALREADY_SEND_EMAIL.txt'
_OUTLOG = 'OUTLOG'
_TRASH = 'TRASH'
_PBS = 'PBS'
_FLAG_FILES = 'FlagFiles'
_UPLOAD_SKIP_LIST = [_OUTLOG, _TRASH, _PBS, _FLAG_FILES]
_SMTP_HOST = 'smtp.gmail.com'

def parse_args():
    from optparse import OptionParser
    usage = "usage: %prog [options] \nWhat is the script doing : Upload Data on Xnat from a Directory as an Assessor. "
    parser = OptionParser(usage=usage)
    parser.add_option("-e", "--emailaddress", dest="emailaddress",default='',
                  help="Email address to prevent if an assessor already exists.", metavar="EMAIL ADDRESS")
    return parser.parse_args()

def sendMail(FROM,PWS,TO,SUBJECT,TEXT,SERVER,filename='nan'):
    """send an email from FROM (with the password PWS) to TO with the subject and text given.
    
    parameters:
        - FROM = email address from where the e-amil is sent
        - PWS = password of the email address
        - TO = list of email address which will receive the email
        - SUBJECT =  subject of the email
        - TEXT = inside of the email
        - server = server used to send the email
        - filename = fullpath to a file that need to be attached
    """
    # Create the container (outer) email message.
    msg = MIMEText(TEXT)
    msg['Subject'] = SUBJECT
    # me == the sender's email address
    # family = the list of all recipients' email addresses
    msg['From'] = FROM
    msg['To'] = TO
    
    #attached the file if one :
    if filename!='nan':
        part = MIMEBase('application', "octet-stream")
        part.set_payload( open(filename,"rb").read() )
        Encoders.encode_base64(part)
        part.add_header('Content-Disposition', 'attachment; filename="%s"' % os.path.basename(filename))
        msg.attach(part)
    
    # Send the email via our own SMTP server.
    s = smtplib.SMTP(SERVER)
    s.starttls()
    s.login(FROM,PWS)
    s.sendmail(FROM, TO, msg.as_string())
    s.quit()

def get_assessor_name_from_folder():
    #Get the assessor label from the folders in Upload Directory that are ready to be upload
    
    #list of assessor label
    assessor_label_list = list()
    
    print ' -Get Processes names from the upload folder...'
    #check all files/folder in the directory
    UploadDirList = os.listdir(RESULTS_DIR)
    for assessor_label in UploadDirList:
        
        if assessor_label in _UPLOAD_SKIP_LIST:
            continue
        
        assessor_path = os.path.join(RESULTS_DIR,assessor_label)
        
        if not os.path.isdir(assessor_path):
            continue
        
        if os.path.exists(assessor_path+'/'+_EMAILED_FLAG_FILE):
            continue
                
        if os.path.exists(os.path.join(assessor_path, _READY_FLAG_FILE)) or os.path.exists(os.path.join(assessor_path, _FAILED_FLAG_FILE)):
            # Passed all checks, so add it to upload list
            assessor_label_list.append(assessor_label)
            
    return assessor_label_list

def get_outlog_from_folder():
    #Get the outlog files which need to be upload
    
    #list of assessor label
    outlog_list=list()
    
    print ' -Get the OUTLOG for the processes...'
    #check all files/folder in the directory
    OutlogDirList=os.listdir(os.path.join(RESULTS_DIR, _OUTLOG))
    for outlog_name in OutlogDirList:
        outlog_file=os.path.join(RESULTS_DIR, _OUTLOG, outlog_name)
        #if it's a folder and not OUTLOG and the flag file READY_TO_UPLAOD exists
        if os.path.isfile(outlog_file):
            #Add the file to the list to be uploaded:
            outlog_list.append(outlog_name)
            
    return outlog_list

def get_pbs_from_folder():
    #Get the pbs files which need to be upload
    
    #list of assessor label
    pbs_list=list()
    
    print ' -Get the PBS for the processes...'
    #check all files/folder in the directory
    PBSDirList=os.listdir(os.path.join(RESULTS_DIR, _PBS))
    for pbs_name in PBSDirList:
        pbs_file=os.path.join(RESULTS_DIR, _PBS,pbs_name)
        #if it's a folder and not OUTLOG and the flag file READY_TO_UPLAOD exists
        if os.path.isfile(pbs_file):
            #add the file to the list to be uploaded
            pbs_list.append(pbs_name)
            
    return pbs_list

def check_process(process_file):
    if process_file=='Process_Upload_running.txt':
        return 0
    elif 'OPEN_TASKS_UPDATE' in process_file:
        project_name=process_file.split('_OPEN_TASKS_UPDATE_RUNNING')[0]
        cmd = 'ps -aux | grep "dax_update_open_tasks" | grep "'+project_name+'"'
    else:
        project_name=process_file.split('_UPDATE_RUNNING')[0]
        cmd = 'ps -aux | grep "dax_update" | grep "'+project_name+'"'
    
    try:
        output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
        if len(output.split('\n'))>=3:
            #three because one is warning, two is the line we just do (ps -aux ...) and three is empty line
            #there is a process running
            return 0
        else:
            #no process, send an email
            return 1
    except (CalledProcessError,ValueError):
        #error , send an email
        return 1

def check_crontab_job():
    flag_files_list=list()
    for files in os.listdir(os.path.join(RESULTS_DIR, _FLAG_FILES)):
        print '   - Check '+files
        #check if there is a process for this file, if not send a warning to the User
        keep=check_process(files)
        if keep:
            print "    --> Process doesn't seem to be running. ERROR. Need to be checked."
            flag_files_list.append(files)
            
    return flag_files_list
            
def Uploading_Assessor(xnat,assessor_path,ProjectName,Subject,Experiment,assessor_label,version):
    #SNAPSHOTS :
    #Check if the SNAPSHOTS folder exists, if not create one from PDF if pdf exists :
    if not os.path.exists(assessor_path+'/SNAPSHOTS/') and os.path.exists(assessor_path+'/PDF/'):
        print '    +creating original of SNAPSHOTS'
        os.system('mkdir '+assessor_path+'/SNAPSHOTS/')
        #Make the snapshots for the assessors with ghostscript
        snapshot_original = assessor_path+'/SNAPSHOTS/snapshot_original.png'
        os.system('gs -q -o '+snapshot_original+' -sDEVICE=pngalpha -dLastPage=1 '+assessor_path+'/PDF/*.pdf')
    
    #Create the preview snapshot from the original if Snapshots exist :
    if os.path.exists(assessor_path+'/SNAPSHOTS/'):
        Assessor_Resource_List=os.listdir(assessor_path+'/SNAPSHOTS/')
        for snapshot in Assessor_Resource_List:
            if len(snapshot.split('original'))>1:
                print '    +creating preview of SNAPSHOTS'
                #Name of the preview snapshot
                snapshot_preview = assessor_path+'/SNAPSHOTS/preview.'+snapshot.split('.')[1]
                #Make the snapshot_thumbnail
                os.system('convert '+assessor_path+'/SNAPSHOTS/'+snapshot+' -resize x200 '+snapshot_preview)                  
    
    #Select the assessor
    assessor = xnat.select('/project/'+ProjectName+'/subjects/'+Subject+'/experiments/'+Experiment+'/assessors/'+assessor_label)
    
    #set version:
    assessor.attrs.set('proc:genProcData/procversion', version)
    
    #UPLOAD files :                
    Assessor_Resource_List = os.listdir(assessor_path)    
    #for each folder=resource in the assessor directory 
    for Resource in Assessor_Resource_List:
        Resource_path = assessor_path+'/'+Resource
        
        #Need to be in a folder to create the resource :
        if os.path.isdir(Resource_path):
            print '    +uploading '+Resource
            #check if the resource exist, if yes remove it
            if assessor.out_resource(Resource).exists():
                assessor.out_resource(Resource).delete()
            
            #create the resource
            r = assessor.out_resource(Resource)
                
            #if it's the SNAPSHOTS folder, need to set the thumbnail and original:
            if Resource == 'SNAPSHOTS':
                #for each files in this folderl, Upload files in the resource :
                Resource_files_list = os.listdir(Resource_path)
                
                preview_list = [s for s in Resource_files_list if "preview" in s]
                if preview_list:
                    preview = preview_list[0]
                    r.file(preview).put(Resource_path+'/'+preview,(preview.split('.')[1]).upper(),'THUMBNAIL')
                    os.remove(Resource_path+'/'+preview)
                    
                original_list = [s for s in Resource_files_list if "original" in s]
                if original_list:
                    original = original_list[0]
                    r.file(original).put(Resource_path+'/'+original,(original.split('.')[1]).upper(),'ORIGINAL')
                    os.remove(Resource_path+'/'+original)
                
                if len(os.listdir(Resource_path)) > 0:
                    upload_zip(Resource,Resource_path,r)
                    
            #for all the other resources :
            else:
                #for each files in this folderl, Upload files in the resource :
                Resource_files_list=os.listdir(Resource_path)
                #for each folder=resource in the assessor directory, more than 2 files, use the zip from XNAT
                if len(Resource_files_list)>2:
                    upload_zip(Resource,Resource_path,r)
                #One or two file, let just upload them:
                else:
                    for filename in Resource_files_list:
                        #if it's a folder, zip it and upload it
                        if os.path.isdir(filename):
                            upload_zip(filename,Resource_path+'/'+filename,r)
                        elif filename.lower().endswith('.zip'):
                            r.put_zip(Resource_path+'/'+filename, extract=True)
                        else:
                            #upload the file
                            r.file(filename).put(Resource_path+'/'+filename)
                        
    #upload finish
    if os.path.exists(os.path.join(assessor_path, _READY_FLAG_FILE)):
        assessor.attrs.set('proc:genProcData/procstatus', READY_TO_COMPLETE)
        
    os.system('rm -r '+assessor_path)

def Uploading_OUTLOG(outlog_list,xnat):
    number_outlog=len(outlog_list)
    for index,outlogfile in enumerate(outlog_list):
        print'   *Uploading OUTLOG '+str(index+1)+'/'+str(number_outlog)+' -- File name: '+outlogfile
        #Get the Project Name, the subject label, the experiment label and the assessor label from the file name :
        labels=outlogfile.split('-x-')
        ProjectName=labels[0]
        Subject=labels[1]
        Experiment=labels[2]
        assessor_label=outlogfile.split('.')[0]
        
        # Connect to experiment
        experiment = xnat.select('/project/'+ProjectName+'/subjects/'+Subject+'/experiments/'+Experiment)
        if not experiment.exists():
            print('ERROR:OUTLOG file '+outlogfile+' has invalid name, moving to TRASH.')
            os.rename(os.path.join(RESULTS_DIR, _OUTLOG, outlogfile),os.path.join(RESULTS_DIR, _TRASH, outlogfile))
            continue
        
        # Connect to assessor
        assessor=experiment.assessor(assessor_label)
        if not assessor.exists():
            print 'ERROR: The assessor '+assessor_label+' does not exist. Moving OUTLOG to TRASH.'        
            os.rename(os.path.join(RESULTS_DIR, _OUTLOG, outlogfile),os.path.join(RESULTS_DIR, _TRASH, outlogfile))  
            continue
        
        # Connect to resource
        r = assessor.out_resource(_OUTLOG)
        if r.exists():
            print 'WARNING: the OUTLOG resource already exists for the assessor '+assessor_label
            #check if there is a folder with the same name : if yes, put the outlog there. If not upload it.
            if  os.path.isdir(os.path.join(RESULTS_DIR,assessor_label)):
                print 'WARNING: Copying the outlog file in the assessor folder...'
                assessor_outlog_folder=os.path.join(RESULTS_DIR,assessor_label, _OUTLOG)
                if not os.path.exists(assessor_outlog_folder):
                    os.mkdir(assessor_outlog_folder)
                os.system('mv '+os.path.join(RESULTS_DIR, _OUTLOG, outlogfile)+' '+assessor_outlog_folder)
            else:
                print 'WARNING: Copying the outlog file in the TRASH ...'
                os.rename(os.path.join(RESULTS_DIR, _OUTLOG, outlogfile),os.path.join(RESULTS_DIR, _TRASH, outlogfile))
        else:
            #upload the file
            r=assessor.out_resource(_OUTLOG)
            r.file(outlogfile).put(os.path.join(RESULTS_DIR, _OUTLOG, outlogfile))
            os.remove(os.path.join(RESULTS_DIR, _OUTLOG, outlogfile))                      

def Uploading_PBS(pbs_list,xnat):
    number_pbs=len(pbs_list)
    for index,pbsfile in enumerate(pbs_list):
        print'   *Uploading PBS '+str(index+1)+'/'+str(number_pbs)+' -- File name: '+pbsfile
        #Get the Project Name, the subject label, the experiment label and the assessor label from the file name :
        labels=pbsfile.split('-x-')
        ProjectName=labels[0]
        Subject=labels[1]
        Experiment=labels[2]
        assessor_label=pbsfile.split('.')[0]
        
        #connect to the experiment
        experiment = xnat.select('/project/'+ProjectName+'/subjects/'+Subject+'/experiments/'+Experiment)
        if experiment.exists():
            #ASSESSOR
            assessor=experiment.assessor(assessor_label)
            #if the assessors doesn't exist send an email
            if assessor.exists():
                #remove the previous resource if exists
                r=assessor.out_resource(_PBS)
                if r.exists():
                    print 'WARNING : the PBS resource already exists for the assessor '+assessor_label
                    #check if there is a folder with the same name : if yes, put the outlog there. If not upload it.
                    if  os.path.isdir(os.path.join(RESULTS_DIR,assessor_label)):
                        print 'WARNING: Copying the pbs file in the assessor folder...'
                        assessor_pbs_folder=os.path.join(RESULTS_DIR,assessor_label, _PBS)
                        if not os.path.exists(assessor_pbs_folder):
                            os.mkdir(assessor_pbs_folder)
                        os.system('mv '+os.path.join(RESULTS_DIR, _PBS, pbsfile)+' '+assessor_pbs_folder)
                    else:
                        print 'WARNING: Copying the pbs file in the TRASH ...'
                        os.rename(os.path.join(RESULTS_DIR, _PBS, pbsfile),os.path.join(RESULTS_DIR, _TRASH, pbsfile))
                else:
                    #upload the file
                    r=assessor.out_resource(_PBS)
                    r.file(pbsfile).put(RESULTS_DIR+'/PBS/'+pbsfile)
                    os.remove(RESULTS_DIR+'/PBS/'+pbsfile)
            else:
                print 'ERROR: The assessor '+assessor_label+' does not exist. Copy to TRASH.'
                os.rename(RESULTS_DIR+'/PBS/'+pbsfile,RESULTS_DIR+'/TRASH/'+pbsfile)
                
        else:
            print 'ERROR: The PBS file '+pbsfile+' has a wrong ProjectName or Subject label or Experiment label in his name. Copy to TRASH.'
            os.rename(RESULTS_DIR+'/PBS/'+pbsfile,RESULTS_DIR+'/TRASH/'+pbsfile)
    
def Upload_FreeSurfer(xnat,assessor_path,ProjectName,Subject,Experiment,assessor_label,version):
    #SNAPSHOTS :
    #Check if the snapshot exists, if not create one from PDF if pdf exists :
    snapshot_original = assessor_path+'/SNAPSHOTS/snapshot_original.png'
    if not os.path.exists(snapshot_original) and os.path.exists(assessor_path+'/PDF/'):
        print '    +creating original of SNAPSHOTS'
        #Make the snapshots for the assessors with ghostscript
        os.system('gs -q -o '+snapshot_original+' -sDEVICE=pngalpha -dLastPage=1 '+assessor_path+'/PDF/*.pdf')
    
    #Create the preview snapshot from the original if Snapshots exist :
    if os.path.exists(assessor_path+'/SNAPSHOTS/'):
        Assessor_Resource_List=os.listdir(assessor_path+'/SNAPSHOTS/')
        for snapshot in Assessor_Resource_List:
            if len(snapshot.split('original'))>1:
                print '    +creating preview of SNAPSHOTS'
                #Name of the preview snapshot
                snapshot_preview = assessor_path+'/SNAPSHOTS/preview.'+snapshot.split('.')[1]
                #Make the snapshot_thumbnail
                os.system('convert '+assessor_path+'/SNAPSHOTS/'+snapshot+' -resize x200 '+snapshot_preview)                  
    
    #Select the experiment
    experiment = xnat.select('/project/'+ProjectName+'/subjects/'+Subject+'/experiments/'+Experiment)
        
    #Select the assessor
    assessor = experiment.assessor(assessor_label)
    
    # set version:
    assessor.attrs.set('fs:fsData/procversion', version)
    # Upload the XML
    xmlpath=os.path.join(assessor_path,'XML')
    if os.path.exists(xmlpath):
        print '    +uploading XML'
        xml_files_list = os.listdir(xmlpath)
        if len(xml_files_list) != 1:
            print 'ERROR:cannot upload FreeSufer, unable to find XML file:'+assessor_path
            return
        xml_path = assessor_path+'/XML/'+xml_files_list[0]
        assessor.create(xml=xml_path, allowDataDeletion=False)
    
    #UPLOAD files :                
    Assessor_Resource_List=os.listdir(assessor_path)    
    #for each folder=resource in the assessor directory 
    for Resource in Assessor_Resource_List:
        Resource_path=assessor_path+'/'+Resource
        
        #Need to be in a folder to create the resource :
        if os.path.isdir(Resource_path):
            print '    +uploading '+Resource
            #check if the resource exist, if yes remove it
            if assessor.out_resource(Resource).exists():
                assessor.out_resource(Resource).delete()
            
            #create the resource
            r = assessor.out_resource(Resource)
                
            #if it's the SNAPSHOTS folder, need to set the thumbnail and original:
            if Resource=='SNAPSHOTS':
                #for each files in this folderl, Upload files in the resource :
                Resource_files_list=os.listdir(Resource_path)
                
                preview_list = [s for s in Resource_files_list if "preview" in s]
                if preview_list:
                    preview = preview_list[0]
                    r.file(preview).put(Resource_path+'/'+preview,(preview.split('.')[1]).upper(),'THUMBNAIL')
                    os.remove(Resource_path+'/'+preview)

                original_list = [s for s in Resource_files_list if "original" in s]
                if original_list:
                    original = original_list[0]
                    r.file(original).put(Resource_path+'/'+original,(original.split('.')[1]).upper(),'ORIGINAL')
                    os.remove(Resource_path+'/'+original)
                
                if len(os.listdir(Resource_path))>2:
                    upload_zip(Resource,Resource_path,r)
              
            #for all the other resources :
            else:
                #for each files in this folderl, Upload files in the resource :
                Resource_files_list=os.listdir(Resource_path)
                
                #for each folder=resource in the assessor directory, more than 2 files, use the zip from XNAT
                if len(Resource_files_list)>2:
                    upload_zip(Resource,Resource_path,r)
                #One or two file, let just upload them:
                else:
                    for filename in Resource_files_list:
                        #if it's a folder, zip it and upload it
                        if os.path.isdir(filename):
                            upload_zip(filename,Resource_path+'/'+filename,r)
                        elif filename.lower().endswith('.zip'):
                            r.put_zip(Resource_path+'/'+filename, extract=True)
                        else:
                            #upload the file
                            r.file(filename).put(Resource_path+'/'+filename)
      
    #upload finish
    if os.path.exists(os.path.join(assessor_path,_READY_FLAG_FILE)):
        assessor.attrs.set('fs:fsdata/procstatus',READY_TO_COMPLETE)
    os.system('rm -r '+assessor_path)
    
def upload_zip(Resource,directory,resourceObj):
    filenameZip=Resource+'.zip'
    initDir=os.getcwd()
    #Zip all the files in the directory
    os.chdir(directory)
    os.system('zip '+filenameZip+' *')
    #upload
    print '      *Uploading zip '+Resource+' ...'
    resourceObj.put_zip(directory+'/'+filenameZip,extract=True)
    #return to the initial directory:
    os.chdir(initDir)
    
def get_version_assessor(assessor_path):
    f=open(os.path.join(assessor_path,'version.txt'),'r')
    version=f.read().strip()
    f.close()
    return version
    
#########################################################################################################################################################
###############################################################  MAIN FUNCTION ##########################################################################
#########################################################################################################################################################

if __name__ == '__main__':
    
    ################### Script for Upload FILES on Assessor on Xnat ######################
    (options,args) = parse_args()
    emailaddress = options.emailaddress
    flag_files_list = list()
    warning_list = list()
    TEXT = None
    
    print 'Time at the beginning of the Process_Upload: ', str(datetime.now()),'\n'

    #Upload Directory for Spider_Process_Upload.py
    try:
        # Environs
        XNAT_USER = os.environ['XNAT_USER']
        XNAT_PWD = os.environ['XNAT_PASS']
        XNAT_HOST = os.environ['XNAT_HOST']
        EMAIL_ADDR = os.environ['EMAIL_ADDR']
        EMAIL_PWS = os.environ['EMAIL_PWS']
    except KeyError as e:
        print "ERROR:you must set the environment variable %s" % str(e)
        sys.exit(1) 
    
    #make the two special directory
    if not os.path.exists(RESULTS_DIR):
        os.mkdir(RESULTS_DIR)
    if not os.path.exists(os.path.join(RESULTS_DIR, _OUTLOG)):
        os.mkdir(os.path.join(RESULTS_DIR, _OUTLOG))
    if not os.path.exists(os.path.join(RESULTS_DIR, _TRASH)):
        os.mkdir(os.path.join(RESULTS_DIR,_TRASH))
    if not os.path.exists(os.path.join(RESULTS_DIR, _PBS)):
        os.mkdir(os.path.join(RESULTS_DIR, _PBS))
    if not os.path.exists(os.path.join(RESULTS_DIR,_FLAG_FILES)):
        os.mkdir(os.path.join(RESULTS_DIR, _FLAG_FILES))
    
    if os.path.exists(os.path.join(RESULTS_DIR, _FLAG_FILES, 'Process_Upload_running.txt')):
        print 'WARNING: Upload already running.'
        sys.exit()

    #create the flag file showing that the spider is running 
    f = open(os.path.join(RESULTS_DIR, _FLAG_FILES, 'Process_Upload_running.txt'), 'w')
    today = datetime.now()
    datestr = "Date: "+str(today.year)+str(today.month)+str(today.day)+'_'+str(today.hour)+':'+str(today.minute)+':'+str(today.second)
    f.write(datestr+'\n')
    f.close()
    
    try:
        #Start Uploading
        print '-------- Upload Directory: '+RESULTS_DIR+' --------'
        ###VARIABLES###
        #Check if the folder is not empty
        UploadDirList = os.listdir(RESULTS_DIR)
        if len(UploadDirList) == 0:
            print('WARNING: No data need to be uploaded.\n')
            sys.exit()
        
        #Get the assessor label from the directory :
        assessor_label_in_dir_list=get_assessor_name_from_folder()
        #Get the list of OUTLOG which need to be upload:
        outlog_list=get_outlog_from_folder()
        #Get the list of OUTLOG which need to be upload:
        pbs_list=get_pbs_from_folder()

        #Start the process to upload
        try:
            print 'INFO: Connecting to XNAT to start uploading processes at '+XNAT_HOST
            xnat = Interface(XNAT_HOST, XNAT_USER, XNAT_PWD)
            
            ################# 1) Upload the assessor data ###############
            #For each assessor label that need to be upload :
            number_of_processes=len(assessor_label_in_dir_list)
            for index,assessor_label in enumerate(assessor_label_in_dir_list):
                assessor_path=RESULTS_DIR+'/'+assessor_label

                if not os.path.isdir(assessor_path):
                    print 'ERROR: The folder '+assessor_label+' has a wrong ProjectName or Subject label or Experiment label.'
                    continue

                sys.stdout.flush()
                sys.stdout.write("    *Process: "+str(index+1)+"/"+str(number_of_processes)+' -- label: '+assessor_label+' / time: '+str(datetime.now())+'\n')
                
                #Get the Project Name, the subject label, the experiment label and the assessor label from the folder name :
                labels=assessor_label.split('-x-')
                ProjectName=labels[0]
                Subject=labels[1]
                Experiment=labels[2]
                #The Process name is the last labels
                Process_name=labels[-1]
                #get spiderpath from version.txt file:
                version = get_version_assessor(assessor_path)
                
                #check if subject/experiment exists on XNAT
                EXPERIMENT = xnat.select('/project/'+ProjectName+'/subjects/'+Subject+'/experiments/'+Experiment)
                
                if EXPERIMENT.exists():
                    #ASSESSOR in the experiment
                    ASSESSOR=EXPERIMENT.assessor(assessor_label)
                    
                    #existence :
                    if not ASSESSOR.exists():
                        if Process_name=='FS':
                            #create the assessor and set the status 
                            ASSESSOR.create(assessors='fs:fsData')
                            #Set attributes
                            if os.path.exists(os.path.join(assessor_path, _READY_FLAG_FILE)):
                                ASSESSOR.attrs.set('fs:fsData/procstatus', UPLOADING) #Set to uploading files
                            elif os.path.exists(os.path.join(assessor_path, _FAILED_FLAG_FILE)):
                                ASSESSOR.attrs.set('fs:fsData/procstatus', JOB_FAILED) #Set to uploading files
                            ASSESSOR.attrs.set('fs:fsData/validation/status', JOB_PENDING)
                            ASSESSOR.attrs.set('fs:fsData/proctype', 'FreeSurfer_v'+version.split('.')[0])
                            ASSESSOR.attrs.set('fs:fsData/procversion', version)
                            now=datetime.now()
                            today=str(now.year)+'-'+str(now.month)+'-'+str(now.day)
                            ASSESSOR.attrs.set('fs:fsData/date',today)
                        else:
                            #create the assessor and set the status 
                            ASSESSOR.create(assessors='proc:genProcData')
                            #Set attributes
                            if os.path.exists(os.path.join(assessor_path,_READY_FLAG_FILE)):
                                ASSESSOR.attrs.set('proc:genProcData/procstatus', UPLOADING) #Set to uploading files
                            elif os.path.exists(os.path.join(assessor_path, _FAILED_FLAG_FILE)):
                                ASSESSOR.attrs.set('proc:genProcData/procstatus', JOB_FAILED) #Set to uploading files
                            ASSESSOR.attrs.set('proc:genProcData/validation/status',JOB_PENDING)
                            ASSESSOR.attrs.set('proc:genProcData/proctype', Process_name)
                            ASSESSOR.attrs.set('proc:genProcData/procversion', version)
                            now=datetime.now()
                            today=str(now.year)+'-'+str(now.month)+'-'+str(now.day)
                            ASSESSOR.attrs.set('proc:genProcData/date',today)
                    
                    else:
                        ################# FreeSurfer #################
                        if Process_name=='FS':      
                            if ASSESSOR.attrs.get('fs:fsData/procstatus') == READY_TO_COMPLETE or ASSESSOR.attrs.get('fs:fsData/procstatus') == COMPLETE:
                                if not os.path.exists(assessor_path+'/'+_EMAILED_FLAG_FILE):
                                    open(assessor_path+'/'+_EMAILED_FLAG_FILE, 'w').close()
                                print '  -->Data already present on XNAT.\n'
                                warning_list.append('\t- Project : '+ProjectName+' / Subject : '+Subject+' / Experiment : '+Experiment+' / Assessor label : '+ assessor_label+'\n')
                            else:
                                #set the status to Upload :
                                if os.path.exists(os.path.join(assessor_path, _READY_FLAG_FILE)):
                                    ASSESSOR.attrs.set('fs:fsData/procstatus', UPLOADING)
                                Upload_FreeSurfer(xnat,assessor_path,ProjectName,Subject,Experiment,assessor_label,version)
                        ################# Default Assessor #################
                        else:
                            if ASSESSOR.attrs.get('proc:genProcData/procstatus') == READY_TO_COMPLETE or ASSESSOR.attrs.get('proc:genProcData/procstatus') == COMPLETE:
                                if not os.path.exists(assessor_path+'/'+_EMAILED_FLAG_FILE):
                                    open(assessor_path+'/'+_EMAILED_FLAG_FILE, 'w').close()
                                print 'Data already exist.\n'
                                warning_list.append('\t- Project : '+ProjectName+' / Subject : '+Subject+' / Experiment : '+Experiment+' / Assessor label : '+ assessor_label+'\n')
                            else:
                                #set the status to Upload :
                                if os.path.exists(os.path.join(assessor_path, _READY_FLAG_FILE)):
                                    ASSESSOR.attrs.set('proc:genProcData/procstatus', UPLOADING)
                                Uploading_Assessor(xnat,assessor_path,ProjectName,Subject,Experiment,assessor_label,version)
              
            ################# 2) Upload the Outlog files ###############
            #For each file, upload it to the OUTLOG resource on the assessor
            print ' - Uploading OUTLOG files ...'
            Uploading_OUTLOG(outlog_list,xnat)
            
            ################# 3) Upload the PBS files ###############
            #For each file, upload it to the PBS resource
            print ' - Uploading PBS files ...'
            Uploading_PBS(pbs_list,xnat)
            
            ################# 4) Check flagfile and process running ps -aux ################
            print ' - Checking process running ...'
            flag_files_list=check_crontab_job()
            
        #if fail, close the connection to xnat
        finally:
            #Sent an email
            if (warning_list or flag_files_list)  and emailaddress!='' :
                if warning_list:
                    TEXT='\nThe following assessor already exists and the Spider try to upload files on existing files :\n'
                    for warning in warning_list:
                        TEXT+=' - '+warning+'\n'
                    TEXT+='\nYou should :\n\t-remove the assessor if you want to upload the data \n\t-set the status of the assessor to "uploading" \n\t-remove the data from the upload folder if you do not want to upload this data.\n'
                SUBJECT='ERROR/WARNING: XNAT Process Upload'
                if flag_files_list:
                    if not TEXT:
                        TEXT='\nCheck the following processes if they are still running:\n'
                    else:
                        TEXT+='Check the following process if they are still running:\n'
                    for files in flag_files_list:
                        TEXT+=' - '+files+'\n'
                sendMail(EMAIL_ADDR, EMAIL_PWS, emailaddress, SUBJECT,TEXT, _SMTP_HOST)   
                
            #disconnect                                     
            xnat.disconnect()
            print 'INFO: Connection to Xnat closed'  

    finally:
        #remove flagfile
        os.remove(os.path.join(RESULTS_DIR,_FLAG_FILES,'Process_Upload_running.txt'))
        print '===================================================================\n'
