""" This is the RFIO StorageClass
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

__RCSID__ = "$Id$"

import re
import os
import time

import six

from DIRAC import gLogger, S_OK, S_ERROR
from DIRAC.Resources.Storage.Utilities import checkArgumentFormat
from DIRAC.Resources.Storage.StorageBase import StorageBase
from DIRAC.Core.Utilities.Subprocess import shellCall
from DIRAC.Core.Utilities.List import breakListIntoChunks
from DIRAC.Core.Utilities.File import getSize


class RFIOStorage(StorageBase):

  _INPUT_PROTOCOLS = ['file', 'rfio']
  _OUTPUT_PROTOCOLS = ['rfio']

  def __init__(self, storageName, parameters):

    StorageBase.__init__(self, storageName, parameters)
    self.spaceToken = self.protocolParameters['SpaceToken']

    self.isok = True

    self.pluginName = 'RFIO'

    self.timeout = 100
    self.long_timeout = 600

  #############################################################
  #
  # These are the methods for manipulating the client
  #

  def getName(self):
    """ The name with which the storage was instantiated
    """
    return S_OK(self.name)

  #############################################################
  #
  # These are the methods for file manipulation
  #

  def exists(self, path):
    """ Check if the given path exists. The 'path' variable can be a string or a list of strings.
    """
    res = self.__checkArgumentFormat(path)
    if not res['OK']:
      return res
    urls = res['Value']
    gLogger.debug("RFIOStorage.exists: Determining the existance of %s files." % len(urls))
    comm = "nsls -d"
    for url in urls:
      comm = " %s %s" % (comm, url)
    res = shellCall(self.timeout, comm)
    successful = {}
    failed = {}
    if res['OK']:
      returncode, stdout, stderr = res['Value']
      if returncode in [0, 1]:
        for line in stdout.splitlines():
          url = line.strip()
          successful[url] = True
        for line in stderr.splitlines():
          pfn, _ = line.split(': ')
          url = pfn.strip()
          successful[url] = False
      else:
        errStr = "RFIOStorage.exists: Completely failed to determine the existance files."
        gLogger.error(errStr, "%s %s" % (self.name, stderr))
        return S_ERROR(errStr)
    else:
      errStr = "RFIOStorage.exists: Completely failed to determine the existance files."
      gLogger.error(errStr, "%s %s" % (self.name, res['Message']))
      return S_ERROR(errStr)
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  def isFile(self, path):
    """Check if the given path exists and it is a file
    """
    res = self.__checkArgumentFormat(path)
    if not res['OK']:
      return res
    urls = res['Value']
    gLogger.debug("RFIOStorage.isFile: Determining whether %s paths are files." % len(urls))
    successful = {}
    failed = {}
    comm = "nsls -ld"
    for url in urls:
      comm = " %s %s" % (comm, url)
    res = shellCall(self.timeout, comm)
    if not res['OK']:
      return res
    returncode, stdout, stderr = res['Value']
    if returncode in [0, 1]:
      for line in stdout.splitlines():
        permissions, _subdirs, _owner, _group, _size, _month, _date, _timeYear, pfn = line.split()
        if permissions[0] != 'd':
          successful[pfn] = True
        else:
          successful[pfn] = False
      for line in stderr.splitlines():
        pfn, error = line.split(': ')
        url = pfn.strip()
        failed[url] = error
    else:
      errStr = "RFIOStorage.isFile: Completely failed to determine whether path is file."
      gLogger.error(errStr, "%s %s" % (self.name, stderr))
      return S_ERROR(errStr)
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  def __getPathMetadata(self, urls):
    gLogger.debug("RFIOStorage.__getPathMetadata: Attempting to get metadata for %s paths." % (len(urls)))
    comm = "nsls -ld"
    for url in urls:
      comm = " %s %s" % (comm, url)
    res = shellCall(self.timeout, comm)
    successful = {}
    failed = {}
    if not res['OK']:
      errStr = "RFIOStorage.__getPathMetadata: Completely failed to get path metadata."
      gLogger.error(errStr, res['Message'])
      return S_ERROR(errStr)
    else:
      returncode, stdout, stderr = res['Value']
      if returncode not in [0, 1]:
        errStr = "RFIOStorage.__getPathMetadata: failed to perform nsls."
        gLogger.error(errStr, stderr)
      else:
        for line in stdout.splitlines():
          permissions, subdirs, owner, group, size, month, date, timeYear, pfn = line.split()
          successful[pfn] = {}
          if permissions[0] == 'd':
            successful[pfn]['Type'] = 'Directory'
          else:
            successful[pfn]['Type'] = 'File'
          successful[pfn]['Mode'] = self.__permissionsToInt(permissions)
          successful[pfn]['NbSubDirs'] = subdirs
          successful[pfn]['Owner'] = owner
          successful[pfn]['Group'] = group
          successful[pfn]['Size'] = int(size)
          successful[pfn]['Month'] = month
          successful[pfn]['Date'] = date
          successful[pfn]['Year'] = timeYear
        for line in stderr.splitlines():
          pfn, error = line.split(': ')
          url = pfn.strip()
          failed[url] = error
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  def __permissionsToInt(self, permissions):
    mode = permissions[1:]
    return sum(pow(2, 8 - i) * int(mode[i] != '-') for i in range(0, 9))

  def __getFileMetadata(self, urls):
    gLogger.debug("RFIOStorage.__getPathMetadata: Attempting to get additional metadata for %s files." % (len(urls)))
    # Check whether the files that exist are staged
    comm = "stager_qry -S %s" % self.spaceToken
    successful = {}
    for pfn in urls:
      successful[pfn] = {}
      comm = "%s -M %s" % (comm, pfn)
    res = shellCall(self.timeout, comm)
    if not res['OK']:
      errStr = "RFIOStorage.__getFileMetadata: Completely failed to get cached status."
      gLogger.error(errStr, res['Message'])
      return S_ERROR(errStr)
    else:
      _returncode, stdout, _stderr = res['Value']
      for line in stdout.splitlines():
        pfn = line.split()[0]
        status = line.split()[-1]
        if status in ['STAGED', 'CANBEMIGR']:
          successful[pfn]['Cached'] = True
    for pfn in urls:
      if 'Cached' not in successful[pfn]:
        successful[pfn]['Cached'] = False

    # Now for the files that exist get the tape segment (i.e. whether they have been migrated) and related checksum
    comm = "nsls -lT --checksum"
    for pfn in urls:
      comm = "%s %s" % (comm, pfn)
    res = shellCall(self.timeout, comm)
    if not res['OK']:
      errStr = "RFIOStorage.__getFileMetadata: Completely failed to get migration status."
      gLogger.error(errStr, res['Message'])
      return S_ERROR(errStr)
    else:
      _returncode, stdout, _stderr = res['Value']
      for line in stdout.splitlines():
        pfn = line.split()[-1]
        checksum = line.split()[-2]
        successful[pfn]['Migrated'] = True
        successful[pfn]['Checksum'] = checksum
    for pfn in urls:
      if 'Migrated' not in successful[pfn]:
        successful[pfn]['Migrated'] = False

    # Update all the metadata with the common one
    for lfn in successful:
      successful[lfn] = self._addCommonMetadata(successful[lfn])

    resDict = {'Failed': {}, 'Successful': successful}
    return S_OK(resDict)

  def getFile(self, path, localPath=False):
    """ Get a local copy in the current directory of a physical file specified by its path
    """
    res = checkArgumentFormat(path)
    if not res['OK']:
      return res
    urls = res['Value']

    failed = {}
    successful = {}
    for src_url in urls.keys():
      fileName = os.path.basename(src_url)
      if localPath:
        dest_file = "%s/%s" % (localPath, fileName)
      else:
        dest_file = "%s/%s" % (os.getcwd(), fileName)
      res = self.__getFile(src_url, dest_file)
      if res['OK']:
        successful[src_url] = res['Value']
      else:
        failed[src_url] = res['Message']
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  def __getFile(self, src_url, dest_file):
    """Get a local copy in the current directory of a physical file specified by its path
    """
    if not os.path.exists(os.path.dirname(dest_file)):
      os.makedirs(os.path.dirname(dest_file))
    if os.path.exists(dest_file):
      gLogger.debug("RFIOStorage.getFile: Local file already exists %s. Removing..." % dest_file)
      os.remove(dest_file)
    res = self.__executeOperation(src_url, 'getFileSize')
    if not res['OK']:
      return S_ERROR(res['Message'])
    remoteSize = res['Value']
    MIN_BANDWIDTH = 1024 * 100  # 100 KB/s
    timeout = int(remoteSize / MIN_BANDWIDTH + 300)
    gLogger.debug("RFIOStorage.getFile: Executing transfer of %s to %s" % (src_url, dest_file))
    comm = "rfcp %s %s" % (src_url, dest_file)
    res = shellCall(timeout, comm)
    if res['OK']:
      returncode, _stdout, stderr = res['Value']
      if returncode == 0:
        gLogger.debug('RFIOStorage.__getFile: Got file from storage, performing post transfer check.')
        localSize = getSize(dest_file)
        if localSize == remoteSize:
          gLogger.debug("RFIOStorage.getFile: Post transfer check successful.")
          return S_OK(localSize)
        errorMessage = "RFIOStorage.__getFile: Source and destination file sizes do not match."
        gLogger.error(errorMessage, src_url)
      else:
        errStr = "RFIOStorage.__getFile: Failed to get local copy of file."
        gLogger.error(errStr, stderr)
        errorMessage = "%s %s" % (errStr, stderr)
    else:
      errStr = "RFIOStorage.__getFile: Failed to get local copy of file."
      gLogger.error(errStr, res['Message'])
      errorMessage = "%s %s" % (errStr, res['Message'])
    if os.path.exists(dest_file):
      gLogger.debug("RFIOStorage.getFile: Removing local file %s." % dest_file)
      os.remove(dest_file)
    return S_ERROR(errorMessage)

  def putFile(self, path, sourceSize=0):
    res = checkArgumentFormat(path)
    if not res['OK']:
      return res
    urls = res['Value']
    failed = {}
    successful = {}
    for dest_url, src_file in urls.items():
      res = self.__executeOperation(os.path.dirname(dest_url), 'createDirectory')
      if not res['OK']:
        failed[dest_url] = res['Message']
      else:
        res = self.__putFile(src_file, dest_url, sourceSize)
        if res['OK']:
          successful[dest_url] = res['Value']
        else:
          failed[dest_url] = res['Message']
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  def __putFile(self, src_file, dest_url, sourceSize):
    """Put a copy of the local file to the current directory on the physical storage
    """
    # Pre-transfer check
    res = self.__executeOperation(dest_url, 'exists')
    if not res['OK']:
      gLogger.debug("RFIOStorage.__putFile: Failed to find pre-existance of destination file.")
      return res
    if res['Value']:
      res = self.__executeOperation(dest_url, 'removeFile')
      if not res['OK']:
        gLogger.debug("RFIOStorage.__putFile: Failed to remove remote file %s." % dest_url)
      else:
        gLogger.debug("RFIOStorage.__putFile: Removed remote file %s." % dest_url)
    if not os.path.exists(src_file):
      errStr = "RFIOStorage.__putFile: The source local file does not exist."
      gLogger.error(errStr, src_file)
      return S_ERROR(errStr)
    sourceSize = getSize(src_file)
    if sourceSize == -1:
      errStr = "RFIOStorage.__putFile: Failed to get file size."
      gLogger.error(errStr, src_file)
      return S_ERROR(errStr)

    res = self.__getTransportURL(dest_url)
    if not res['OK']:
      gLogger.debug("RFIOStorage.__putFile: Failed to get transport URL for file.")
      return res
    turl = res['Value']

    MIN_BANDWIDTH = 1024 * 100  # 100 KB/s
    timeout = sourceSize / MIN_BANDWIDTH + 300
    gLogger.debug("RFIOStorage.putFile: Executing transfer of %s to %s" % (src_file, turl))
    comm = "rfcp %s '%s'" % (src_file, turl)
    res = shellCall(timeout, comm)
    if res['OK']:
      returncode, _stdout, stderr = res['Value']
      if returncode == 0:
        gLogger.debug('RFIOStorage.putFile: Put file to storage, performing post transfer check.')
        res = self.__executeOperation(dest_url, 'getFileSize')
        if res['OK']:
          destinationSize = res['Value']
          if sourceSize == destinationSize:
            gLogger.debug("RFIOStorage.__putFile: Post transfer check successful.")
            return S_OK(destinationSize)
        errorMessage = "RFIOStorage.__putFile: Source and destination file sizes do not match."
        gLogger.error(errorMessage, dest_url)
      else:
        errStr = "RFIOStorage.__putFile: Failed to put file to remote storage."
        gLogger.error(errStr, stderr)
        errorMessage = "%s %s" % (errStr, stderr)
    else:
      errStr = "RFIOStorage.__putFile: Failed to put file to remote storage."
      gLogger.error(errStr, res['Message'])
      errorMessage = "%s %s" % (errStr, res['Message'])
    res = self.__executeOperation(dest_url, 'removeFile')
    if res['OK']:
      gLogger.debug("RFIOStorage.__putFile: Removed remote file remnant %s." % dest_url)
    else:
      gLogger.debug("RFIOStorage.__putFile: Unable to remove remote file remnant %s." % dest_url)
    return S_ERROR(errorMessage)

  def removeFile(self, path):
    """Remove physically the file specified by its path
    """
    res = self.__checkArgumentFormat(path)
    if not res['OK']:
      return res
    urls = res['Value']
    successful = {}
    failed = {}
    listOfLists = breakListIntoChunks(urls, 100)
    for urls in listOfLists:
      gLogger.debug("RFIOStorage.removeFile: Attempting to remove %s files." % len(urls))
      comm = 'stager_rm -S %s' % self.spaceToken
      for url in urls:
        comm = "%s -M %s" % (comm, url)
      res = shellCall(100, comm)
      if res['OK']:
        returncode, _stdout, stderr = res['Value']
        if returncode in [0, 1]:
          comm = 'nsrm -f'
          for url in urls:
            comm = "%s %s" % (comm, url)
          res = shellCall(100, comm)
          if res['OK']:
            returncode, _stdout, stderr = res['Value']
            if returncode in [0, 1]:
              for pfn in urls:
                successful[pfn] = True
            else:
              errStr = "RFIOStorage.removeFile. Completely failed to remove files from the nameserver."
              gLogger.error(errStr, stderr)
              for pfn in urls:
                failed[pfn] = errStr
          else:
            errStr = "RFIOStorage.removeFile. Completely failed to remove files from the nameserver."
            gLogger.error(errStr, res['Message'])
            for pfn in urls:
              failed[pfn] = errStr
        else:
          errStr = "RFIOStorage.removeFile. Completely failed to remove files from the stager."
          gLogger.error(errStr, stderr)
          for pfn in urls:
            failed[pfn] = errStr
      else:
        errStr = "RFIOStorage.removeFile. Completely failed to remove files from the stager."
        gLogger.error(errStr, res['Message'])
        for pfn in urls:
          failed[pfn] = errStr
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  def getFileMetadata(self, path):
    """  Get metadata associated to the file
    """
    res = self.__checkArgumentFormat(path)
    if not res['OK']:
      return res
    urls = res['Value']
    gLogger.debug("RFIOStorage.getFileMetadata: Obtaining metadata for %s files." % len(urls))
    res = self.__getPathMetadata(urls)
    if not res['OK']:
      return res
    failed = {}
    successful = {}
    for pfn, error in res['Value']['Failed'].items():
      if error == 'No such file or directory':
        failed[pfn] = 'File does not exist'
      else:
        failed[pfn] = error
    files = []
    for pfn, pfnDict in res['Value']['Successful'].items():
      if pfnDict['Type'] == 'Directory':
        failed[pfn] = "Supplied path is not a file"
      else:
        successful[pfn] = res['Value']['Successful'][pfn]
        files.append(pfn)
    if files:
      res = self.__getFileMetadata(files)
      if not res['OK']:
        return res
      for pfn, pfnDict in res['Value']['Successful'].items():
        successful[pfn].update(pfnDict)
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  def getFileSize(self, path):
    """Get the physical size of the given file
    """
    res = self.__checkArgumentFormat(path)
    if not res['OK']:
      return res
    urls = res['Value']
    gLogger.debug("RFIOStorage.getFileSize: Determining the sizes for  %s files." % len(urls))
    res = self.__getPathMetadata(urls)
    if not res['OK']:
      return res
    failed = {}
    successful = {}
    for pfn, error in res['Value']['Failed'].items():
      if error == 'No such file or directory':
        failed[pfn] = 'File does not exist'
      else:
        failed[pfn] = error
    for pfn, pfnDict in res['Value']['Successful'].items():
      if pfnDict['Type'] == 'Directory':
        failed[pfn] = "Supplied path is not a file"
      else:
        successful[pfn] = res['Value']['Successful'][pfn]['Size']
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  def prestageFile(self, path):
    """ Issue prestage request for file
    """
    res = self.__checkArgumentFormat(path)
    if not res['OK']:
      return res
    urls = res['Value']
    userTag = '%s-%s' % (self.spaceToken, time.time())
    comm = "stager_get -S %s -U %s " % (self.spaceToken, userTag)
    for url in urls:
      comm = "%s -M %s" % (comm, url)
    res = shellCall(100, comm)
    successful = {}
    failed = {}
    if res['OK']:
      returncode, stdout, stderr = res['Value']
      if returncode in [0, 1]:
        for line in stdout.splitlines():
          if re.search('SUBREQUEST_READY', line):
            pfn, _status = line.split()
            successful[pfn] = userTag
          elif re.search('SUBREQUEST_FAILED', line):
            pfn, _status, err = line.split(' ', 2)
            failed[pfn] = err
      else:
        errStr = "RFIOStorage.prestageFile: Got unexpected return code from stager_get."
        gLogger.error(errStr, stderr)
        return S_ERROR(errStr)
    else:
      errStr = "RFIOStorage.prestageFile: Completely failed to issue stage requests."
      gLogger.error(errStr, res['Message'])
      return S_ERROR(errStr)
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  def prestageFileStatus(self, path):
    """ Monitor the status of a prestage request
    """
    res = checkArgumentFormat(path)
    if not res['OK']:
      return res
    urls = res['Value']
    successful = {}
    failed = {}
    requestFiles = {}
    for url, requestID in urls.items():
      if requestID not in requestFiles:
        requestFiles[requestID] = []
      requestFiles[requestID].append(url)
    for requestID, urls in requestFiles.items():
      comm = "stager_qry -S %s -U %s " % (self.spaceToken, requestID)
      res = shellCall(100, comm)
      if res['OK']:
        returncode, stdout, stderr = res['Value']
        if returncode in [0, 1]:
          for line in stdout.splitlines():
            pfn = line.split()[0]
            status = line.split()[-1]
            if status in ['STAGED', 'CANBEMIGR']:
              successful[pfn] = True
            else:
              successful[pfn] = False
        else:
          errStr = "RFIOStorage.prestageFileStatus: Got unexpected return code from stager_get."
          gLogger.error(errStr, stderr)
          return S_ERROR(errStr)
      else:
        errStr = "RFIOStorage.prestageFileStatus: Completely failed to obtain prestage status."
        gLogger.error(errStr, res['Message'])
        return S_ERROR(errStr)
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  def getTransportURL(self, path, protocols=False):
    """ Obtain the TURLs for the supplied path and protocols
    """
    res = self.__checkArgumentFormat(path)
    if not res['OK']:
      return res
    urls = res['Value']
    successful = {}
    failed = {}
    res = self.exists(urls)
    if not res['OK']:
      return res
    for path, exists in res['Value']['Successful'].items():
      if not exists:
        failed[path] = 'File does not exist'
      else:
        res = self.__getTransportURL(path)
        if not res['OK']:
          failed[path] = res['Message']
        else:
          successful[path] = res['Value']
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  def __getTransportURL(self, path):
    try:
      if self.spaceToken:
        tURL = "%s://%s:%s/?svcClass=%s&castorVersion=2&path=%s" % (self.protocolParameters['Protocol'],
                                                                    self.protocolParameters['Host'],
                                                                    self.protocolParameters['Port'],
                                                                    self.spaceToken,
                                                                    path)
      else:
        tURL = "castor:%s" % (path)
      return S_OK(tURL)
    except Exception as x:
      errStr = "RFIOStorage.__getTransportURL: Exception while creating turl."
      gLogger.exception(errStr, self.name, x)
      return S_ERROR(errStr)

  #############################################################
  #
  # These are the methods for directory manipulation
  #

  def isDirectory(self, path):
    """Check if the given path exists and it is a directory
    """
    res = self.__checkArgumentFormat(path)
    if not res['OK']:
      return res
    urls = res['Value']
    gLogger.debug("RFIOStorage.isDirectory: Determining whether %s paths are directories." % len(urls))
    res = self.__getPathMetadata(urls)
    if not res['OK']:
      return res
    failed = {}
    successful = {}
    for pfn, error in res['Value']['Failed'].items():
      if error == 'No such file or directory':
        failed[pfn] = 'Directory does not exist'
      else:
        failed[pfn] = error
    for pfn, pfnDict in res['Value']['Successful'].items():
      if pfnDict['Type'] == 'Directory':
        successful[pfn] = True
      else:
        successful[pfn] = False
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  def getDirectory(self, path, localPath=False):
    """ Get locally a directory from the physical storage together with all its files and subdirectories.
    """
    res = self.__checkArgumentFormat(path)
    if not res['OK']:
      return res
    urls = res['Value']

    successful = {}
    failed = {}
    gLogger.debug("RFIOStorage.getDirectory: Attempting to get local copies of %s directories." % len(urls))
    for src_directory in urls:
      dirName = os.path.basename(src_directory)
      if localPath:
        dest_dir = "%s/%s" % (localPath, dirName)
      else:
        dest_dir = "%s/%s" % (os.getcwd(), dirName)
      res = self.__getDir(src_directory, dest_dir)
      if res['OK']:
        if res['Value']['AllGot']:
          gLogger.debug("RFIOStorage.getDirectory: Successfully got local copy of %s" % src_directory)
          successful[src_directory] = {'Files': res['Value']['Files'], 'Size': res['Value']['Size']}
        else:
          gLogger.error("RFIOStorage.getDirectory: Failed to get entire directory.", src_directory)
          failed[src_directory] = {'Files': res['Value']['Files'], 'Size': res['Value']['Size']}
      else:
        gLogger.error("RFIOStorage.getDirectory: Completely failed to get local copy of directory.", src_directory)
        failed[src_directory] = {'Files': 0, 'Size': 0}
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  def __getDir(self, srcDirectory, destDirectory):
    """ Black magic contained within...
    """
    filesGot = 0
    sizeGot = 0

    # Check the remote directory exists
    res = self.isDirectory(srcDirectory)
    if not res['OK']:
      errStr = "RFIOStorage.__getDir: Failed to find the supplied source directory."
      gLogger.error(errStr, srcDirectory)
      return S_ERROR(errStr)
    if srcDirectory not in res['Value']['Successful']:
      errStr = "RFIOStorage.__getDir: Failed to find the supplied source directory."
      gLogger.error(errStr, srcDirectory)
      return S_ERROR(errStr)
    if not res['Value']['Successful'][srcDirectory]:
      errStr = "RFIOStorage.__getDir: The supplied source directory does not exist."
      gLogger.error(errStr, srcDirectory)
      return S_ERROR(errStr)

    # Check the local directory exists and create it if not
    if not os.path.exists(destDirectory):
      os.makedirs(destDirectory)

    # Get the remote directory contents
    res = self.listDirectory(srcDirectory)
    if not res['OK']:
      errStr = "RFIOStorage.__getDir: Failed to list the source directory."
      gLogger.error(errStr, srcDirectory)
    if srcDirectory not in res['Value']['Successful']:
      errStr = "RFIOStorage.__getDir: Failed to list the source directory."
      gLogger.error(errStr, srcDirectory)

    surlsDict = res['Value']['Successful'][srcDirectory]['Files']
    subDirsDict = res['Value']['Successful'][srcDirectory]['SubDirs']

    # First get all the files in the directory
    gotFiles = True
    for surl in surlsDict.keys():
      surlGot = False
      fileSize = surlsDict[surl]['Size']
      fileName = os.path.basename(surl)
      localPath = '%s/%s' % (destDirectory, fileName)
      fileDict = {surl: localPath}
      res = self.getFile(fileDict)
      if res['OK']:
        if surl in res['Value']['Successful']:
          filesGot += 1
          sizeGot += fileSize
          surlGot = True
      if not surlGot:
        gotFiles = False

    # Then recursively get the sub directories
    subDirsGot = True
    for subDir in subDirsDict.keys():
      subDirName = os.path.basename(subDir)
      localPath = '%s/%s' % (destDirectory, subDirName)
      dirSuccessful = False
      res = self.__getDir(subDir, localPath)
      if res['OK']:
        if res['Value']['AllGot']:
          dirSuccessful = True
        filesGot += res['Value']['Files']
        sizeGot += res['Value']['Size']
      if not dirSuccessful:
        subDirsGot = False

    # Check whether all the operations were successful
    if subDirsGot and gotFiles:
      allGot = True
    else:
      allGot = False
    resDict = {'AllGot': allGot, 'Files': filesGot, 'Size': sizeGot}
    return S_OK(resDict)

  def putDirectory(self, path):
    """ Put a local directory to the physical storage together with all its files and subdirectories.
    """
    res = checkArgumentFormat(path)
    if not res['OK']:
      return res
    urls = res['Value']

    successful = {}
    failed = {}
    gLogger.debug("RFIOStorage.putDirectory: Attemping to put %s directories to remote storage." % len(urls))
    for destDir, sourceDir in urls.items():
      res = self.__putDir(sourceDir, destDir)
      if res['OK']:
        if res['Value']['AllPut']:
          gLogger.debug("RFIOStorage.putDirectory: Successfully put directory to remote storage: %s" % destDir)
          successful[destDir] = {'Files': res['Value']['Files'], 'Size': res['Value']['Size']}
        else:
          gLogger.error("RFIOStorage.putDirectory: Failed to put entire directory to remote storage.", destDir)
          failed[destDir] = {'Files': res['Value']['Files'], 'Size': res['Value']['Size']}
      else:
        gLogger.error("RFIOStorage.putDirectory: Completely failed to put directory to remote storage.", destDir)
        failed[destDir] = {'Files': 0, 'Size': 0}
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  def __putDir(self, src_directory, dest_directory):
    """ Black magic contained within...
    """
    filesPut = 0
    sizePut = 0

    # Check the local directory exists
    if not os.path.isdir(src_directory):
      errStr = "RFIOStorage.__putDir: The supplied source directory does not exist."
      gLogger.error(errStr, src_directory)
      return S_ERROR(errStr)

    # Create the remote directory
    res = self.createDirectory(dest_directory)
    if not res['OK']:
      errStr = "RFIOStorage.__putDir: Failed to create destination directory."
      gLogger.error(errStr, dest_directory)
      return S_ERROR(errStr)

    # Get the local directory contents
    contents = os.listdir(src_directory)
    allSuccessful = True
    for cFile in contents:
      pathSuccessful = False
      localPath = '%s/%s' % (src_directory, cFile)
      remotePath = '%s/%s' % (dest_directory, cFile)
      if os.path.isdir(localPath):
        res = self.__putDir(localPath, remotePath)
        if res['OK']:
          if res['Value']['AllPut']:
            pathSuccessful = True
          filesPut += res['Value']['Files']
          sizePut += res['Value']['Size']
        else:
          return S_ERROR('Failed to put directory')
      else:
        fileDict = {remotePath: localPath}
        res = self.putFile(fileDict)
        if res['OK']:
          if remotePath in res['Value']['Successful']:
            filesPut += 1
            sizePut += res['Value']['Successful'][remotePath]
            pathSuccessful = True
      if not pathSuccessful:
        allSuccessful = False
    resDict = {'AllPut': allSuccessful, 'Files': filesPut, 'Size': sizePut}
    return S_OK(resDict)

  def createDirectory(self, path):
    res = self.__checkArgumentFormat(path)
    if not res['OK']:
      return res
    urls = res['Value']
    successful = {}
    failed = {}
    gLogger.debug("RFIOStorage.createDirectory: Attempting to create %s directories." % len(urls))
    for url in urls:
      strippedUrl = url.rstrip('/')
      res = self.__makeDirs(strippedUrl)
      if res['OK']:
        gLogger.debug("RFIOStorage.createDirectory: Successfully created directory on storage: %s" % url)
        successful[url] = True
      else:
        gLogger.error("RFIOStorage.createDirectory: Failed to create directory on storage.", url)
        failed[url] = res['Message']
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  def __makeDir(self, path):
    # First create a local file that will be used as a directory place holder in storage name space
    comm = "nsmkdir -m 775 %s" % path
    res = shellCall(100, comm)
    if not res['OK']:
      return res
    returncode, _stdout, stderr = res['Value']
    if returncode not in [0]:
      return S_ERROR(stderr)
    return S_OK()

  def __makeDirs(self, path):
    """  Black magic contained within....
    """
    pDir = os.path.dirname(path)
    res = self.exists(path)
    if not res['OK']:
      return res
    if res['OK']:
      if path in res['Value']['Successful']:
        if res['Value']['Successful'][path]:
          return S_OK()
        else:
          res = self.exists(pDir)
          if res['OK']:
            if pDir in res['Value']['Successful']:
              if res['Value']['Successful'][pDir]:
                res = self.__makeDir(path)
              else:
                res = self.__makeDirs(pDir)
                res = self.__makeDir(path)
    return res

  def removeDirectory(self, path, recursive=False):
    """Remove a directory on the physical storage together with all its files and
       subdirectories.
    """
    res = self.__checkArgumentFormat(path)
    if not res['OK']:
      return res
    urls = res['Value']
    gLogger.debug("RFIOStorage.removeDirectory: Attempting to remove %s directories." % len(urls))
    successful = {}
    failed = {}
    for url in urls:
      comm = "nsrm -r %s" % url
      res = shellCall(100, comm)
      if res['OK']:
        returncode, _stdout, stderr = res['Value']
        if returncode == 0:
          successful[url] = {'FilesRemoved': 0, 'SizeRemoved': 0}
        elif returncode == 1:
          successful[url] = {'FilesRemoved': 0, 'SizeRemoved': 0}
        else:
          failed[url] = stderr
      else:
        errStr = "RFIOStorage.removeDirectory: Completely failed to remove directory."
        gLogger.error(errStr, "%s %s" % (url, res['Message']))
        failed[url] = res['Message']
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  def listDirectory(self, path):
    """ List the supplied path. First checks whether the path is a directory then gets the contents.
    """
    res = self.__checkArgumentFormat(path)
    if not res['OK']:
      return res
    urls = res['Value']
    gLogger.debug("RFIOStorage.listDirectory: Attempting to list %s directories." % len(urls))
    res = self.isDirectory(urls)
    if not res['OK']:
      return res
    successful = {}
    failed = res['Value']['Failed']
    directories = []
    for url, isDirectory in res['Value']['Successful'].items():
      if isDirectory:
        directories.append(url)
      else:
        errStr = "RFIOStorage.listDirectory: Directory does not exist."
        gLogger.error(errStr, url)
        failed[url] = errStr

    for directory in directories:
      comm = "nsls -l %s" % directory
      res = shellCall(self.timeout, comm)
      if res['OK']:
        returncode, stdout, stderr = res['Value']
        if not returncode == 0:
          errStr = "RFIOStorage.listDirectory: Failed to list directory."
          gLogger.error(errStr, "%s %s" % (directory, stderr))
          failed[directory] = errStr
        else:
          subDirs = {}
          files = {}
          successful[directory] = {}
          for line in stdout.splitlines():
            permissions, _subdirs, _owner, _group, size, _month, _date, _timeYear, pfn = line.split()
            if not pfn == 'dirac_directory':
              path = "%s/%s" % (directory, pfn)
              if permissions[0] == 'd':
                # If the subpath is a directory
                subDirs[path] = True
              elif permissions[0] == 'm':
                # In the case that the path is a migrated file
                files[path] = {'Size': int(size), 'Migrated': 1}
              else:
                # In the case that the path is not migrated file
                files[path] = {'Size': int(size), 'Migrated': 0}
          successful[directory]['SubDirs'] = subDirs
          successful[directory]['Files'] = files
      else:
        errStr = "RFIOStorage.listDirectory: Completely failed to list directory."
        gLogger.error(errStr, "%s %s" % (directory, res['Message']))
        return S_ERROR(errStr)
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  def getDirectoryMetadata(self, path):
    """ Get the metadata for the directory
    """
    res = self.__checkArgumentFormat(path)
    if not res['OK']:
      return res
    urls = res['Value']
    gLogger.debug("RFIOStorage.getDirectoryMetadata: Attempting to get metadata for %s directories." % len(urls))
    res = self.isDirectory(urls)
    if not res['OK']:
      return res
    successful = {}
    failed = res['Value']['Failed']
    directories = []
    for url, isDirectory in res['Value']['Successful'].items():
      if isDirectory:
        directories.append(url)
      else:
        errStr = "RFIOStorage.getDirectoryMetadata: Directory does not exist."
        gLogger.error(errStr, url)
        failed[url] = errStr
    res = self.__getPathMetadata(directories)
    if not res['OK']:
      return res
    else:
      failed.update(res['Value']['Failed'])
      successful = res['Value']['Successful']
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  def getDirectorySize(self, path):
    """ Get the size of the directory on the storage
    """
    res = self.__checkArgumentFormat(path)
    if not res['OK']:
      return res
    urls = res['Value']
    gLogger.debug("RFIOStorage.getDirectorySize: Attempting to get size of %s directories." % len(urls))
    res = self.listDirectory(urls)
    if not res['OK']:
      return res
    failed = res['Value']['Failed']
    successful = {}
    for directory, dirDict in res['Value']['Successful'].items():
      directorySize = 0
      directoryFiles = 0
      filesDict = dirDict['Files']
      for _fileURL, fileDict in filesDict.items():
        directorySize += fileDict['Size']
        directoryFiles += 1
      gLogger.debug("RFIOStorage.getDirectorySize: Successfully obtained size of %s." % directory)
      successful[directory] = {'Files': directoryFiles, 'Size': directorySize}
    resDict = {'Failed': failed, 'Successful': successful}
    return S_OK(resDict)

  def __checkArgumentFormat(self, path):
    """  FIXME: Can be replaced by a generic checkArgumentFormat Utility
    """
    if isinstance(path, six.string_types):
      urls = [path]
    elif isinstance(path, list):
      urls = path
    elif isinstance(path, dict):
      urls = list(path)
    else:
      return S_ERROR("RFIOStorage.__checkArgumentFormat: Supplied path is not of the correct format.")
    return S_OK(urls)

  def __executeOperation(self, url, method):
    """ Executes the requested functionality with the supplied url
    """
    fcn = None
    if hasattr(self, method) and callable(getattr(self, method)):
      fcn = getattr(self, method)
    if not fcn:
      return S_ERROR("Unable to invoke %s, it isn't a member funtion of RFIOStorage" % method)
    res = fcn(url)
    if not res['OK']:
      return res
    elif url not in res['Value']['Successful']:
      return S_ERROR(res['Value']['Failed'][url])
    return S_OK(res['Value']['Successful'][url])
