""" DIRAC FileCatalog plugin class to manage file metadata
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

__RCSID__ = "$Id$"

import six
from DIRAC import S_OK, S_ERROR
from DIRAC.Core.Utilities.Time import queryTime
from DIRAC.Core.Utilities.List import intListToString
from DIRAC.DataManagementSystem.Client.MetaQuery import FILE_STANDARD_METAKEYS, \
    FILES_TABLE_METAKEYS, \
    FILEINFO_TABLE_METAKEYS


class FileMetadata(object):

  def __init__(self, database=None):

    self.db = database

  def setDatabase(self, database):
    self.db = database

##############################################################################
#
#  Manage Metadata fields
#
##############################################################################
  def addMetadataField(self, pName, pType, credDict):
    """ Add a new metadata parameter to the Metadata Database.

        :param str pName: parameter name
        :param str pType: parameter type in the MySQL notation

        :return: S_OK/S_ERROR, Value - comment on a positive result
    """

    if pName in FILE_STANDARD_METAKEYS:
      return S_ERROR('Illegal use of reserved metafield name')

    result = self.db.dmeta._getMetadataFields(credDict)
    if not result['OK']:
      return result
    if pName in result['Value']:
      return S_ERROR('The metadata %s is already defined for Directories' % pName)
    result = self._getFileMetadataFields(credDict)
    if not result['OK']:
      return result
    if pName in result['Value']:
      if pType.lower() == result['Value'][pName].lower():
        return S_OK('Already exists')
      else:
        return S_ERROR('Attempt to add an existing metadata with different type: %s/%s' %
                       (pType, result['Value'][pName]))
    valueType = pType
    if pType == "MetaSet":
      valueType = "VARCHAR(64)"
    req = "CREATE TABLE FC_FileMeta_%s ( FileID INTEGER NOT NULL, Value %s, PRIMARY KEY (FileID), INDEX (Value) )" \
        % (pName, valueType)
    result = self.db._query(req)
    if not result['OK']:
      return result

    result = self.db.insertFields('FC_FileMetaFields', ['MetaName', 'MetaType'], [pName, pType])
    if not result['OK']:
      return result

    metadataID = result['lastRowId']
    result = self.__transformMetaParameterToData(pName)
    if not result['OK']:
      return result
    return S_OK("Added new metadata: %d" % metadataID)

  def deleteMetadataField(self, pName, credDict):
    """ Remove metadata field

        :param str pName: meta parameter name
        :param dict credDict: client credential dictionary

        :return: S_OK/S_ERROR
    """

    req = "DROP TABLE FC_FileMeta_%s" % pName
    result = self.db._update(req)
    error = ''
    if not result['OK']:
      error = result["Message"]
    req = "DELETE FROM FC_FileMetaFields WHERE MetaName='%s'" % pName
    result = self.db._update(req)
    if not result['OK']:
      if error:
        result["Message"] = error + "; " + result["Message"]
    return result

  def getFileMetadataFields(self, credDict):
    """ Get all the defined metadata fields

        :param dict credDict: client credential dictionary
        :return: S_OK/S_ERROR, Value is the metadata:metadata type dictionary
    """

    return self._getFileMetadataFields(credDict)

  def _getFileMetadataFields(self, credDict):
    """ Get all the defined metadata fields as they are stored in the database

        :param dict credDict: client credential dictionary
        :return: S_OK/S_ERROR, Value is the metadata:metadata type dictionary
    """

    req = "SELECT MetaName,MetaType FROM FC_FileMetaFields"
    result = self.db._query(req)
    if not result['OK']:
      return result

    metaDict = {}
    for row in result['Value']:
      metaDict[row[0]] = row[1]

    return S_OK(metaDict)

###########################################################
#
# Set and get metadata for files
#
###########################################################

  def setMetadata(self, path, metaDict, credDict):
    """ Set the value of a given metadata field for the the given directory path

        :param str path: file path
        :param dict metaDict: dictionary with metadata
        :param dict credDict: client credential dictionary

        :return: S_OK/S_ERROR
    """
    result = self._getFileMetadataFields(credDict)
    if not result['OK']:
      return result
    metaFields = result['Value']

    result = self.db.fileManager._findFiles([path])
    if not result['OK']:
      return result
    if result['Value']['Successful']:
      fileID = result['Value']['Successful'][path]['FileID']
    else:
      return S_ERROR('File %s not found' % path)

    for metaName, metaValue in metaDict.items():
      if metaName not in metaFields:
        result = self.__setFileMetaParameter(fileID, metaName, metaValue, credDict)
      else:
        result = self.db.insertFields('FC_FileMeta_%s' % metaName,
                                      ['FileID', 'Value'],
                                      [fileID, metaValue])
        if not result['OK']:
          if result['Message'].find('Duplicate') != -1:
            req = "UPDATE FC_FileMeta_%s SET Value='%s' WHERE FileID=%d" % (metaName, metaValue, fileID)
            result = self.db._update(req)
            if not result['OK']:
              return result
          else:
            return result

    return S_OK()

  def removeMetadata(self, path, metadata, credDict):
    """ Remove the specified metadata for the given file

        :param str path: file path
        :param list metadata: list of meta parameter names
        :param dict credDict: client credential dictionary

        :return: S_OK/S_ERROR
    """
    result = self._getFileMetadataFields(credDict)
    if not result['OK']:
      return result
    metaFields = result['Value']

    result = self.db.fileManager._findFiles([path])
    if not result['OK']:
      return result
    if result['Value']['Successful']:
      fileID = result['Value']['Successful'][path]['FileID']
    else:
      return S_ERROR('File %s not found' % path)

    failedMeta = {}
    for meta in metadata:
      if meta in metaFields:
        # Indexed meta case
        req = "DELETE FROM FC_FileMeta_%s WHERE FileID=%d" % (meta, fileID)
        result = self.db._update(req)
        if not result['OK']:
          failedMeta[meta] = result['Value']
      else:
        # Meta parameter case
        req = "DELETE FROM FC_FileMeta WHERE MetaKey='%s' AND FileID=%d" % (meta, fileID)
        result = self.db._update(req)
        if not result['OK']:
          failedMeta[meta] = result['Value']

    if failedMeta:
      metaExample = list(failedMeta)[0]
      result = S_ERROR('Failed to remove %d metadata, e.g. %s' % (len(failedMeta), failedMeta[metaExample]))
      result['FailedMetadata'] = failedMeta
    else:
      return S_OK()

  def __getFileID(self, path):
    """ Get file ID for the given file name

        :param str path: file path

        :return: S_OK/S_ERROR, Value - file ID
    """

    result = self.db.fileManager._findFiles([path])
    if not result['OK']:
      return result
    if result['Value']['Successful']:
      fileID = result['Value']['Successful'][path]['FileID']
    else:
      return S_ERROR('File not found')
    return S_OK(fileID)

  def __setFileMetaParameter(self, fileID, metaName, metaValue, credDict):
    """ Set an meta parameter - metadata which is not used in the the data
        search operations

        :param int fileID: file ID
        :param str metaName: meta parameter name
        :param str metaValue: meta parameter value
        :param dict credDict: client credential dictionary

        :return: S_OK/S_ERROR
    """
    result = self.db.insertFields('FC_FileMeta',
                                  ['FileID', 'MetaKey', 'MetaValue'],
                                  [fileID, metaName, str(metaValue)])
    return result

  def setFileMetaParameter(self, path, metaName, metaValue, credDict):
    """ Set an meta parameter - metadata which is not used in the the data
        search operations

        :param str path: file name
        :param str metaName: meta parameter name
        :param str metaValue: meta parameter value
        :param dict credDict: client credential dictionary

        :return: S_OK/S_ERROR
    """

    result = self.__getFileID(path)
    if not result['OK']:
      return result
    fileID = result['Value']
    return self.__setFileMetaParameter(fileID, metaName, metaValue, credDict)

  def _getFileUserMetadataByID(self, fileIDList, credDict, connection=False):
    """ Get file user metadata for the list of file IDs

        :param list fileIDList: list of file IDs (int)
        :param dict credDict: client credential dictionary
        :param obj connection: database connection object

        :return: S_OK/S_ERROR, Value - dict of metadata for each input file
    """
    # First file metadata
    result = self._getFileMetadataFields(credDict)
    if not result['OK']:
      return result
    metaFields = result['Value']

    stringIDs = ','.join(['%s' % fId for fId in fileIDList])
    metaDict = {}
    for meta in metaFields:
      req = "SELECT Value,FileID FROM FC_FileMeta_%s WHERE FileID in (%s)" % (meta, stringIDs)
      result = self.db._query(req, conn=connection)
      if not result['OK']:
        return result
      for value, fileID in result['Value']:
        metaDict.setdefault(fileID, {})
        metaDict[fileID][meta] = value

    req = "SELECT FileID,MetaKey,MetaValue from FC_FileMeta where FileID in (%s)" % stringIDs
    result = self.db._query(req, conn=connection)
    if not result['OK']:
      return result
    for fileID, key, value in result['Value']:
      metaDict.setdefault(fileID, {})
      metaDict[fileID][key] = value

    return S_OK(metaDict)

  def getFileUserMetadata(self, path, credDict):
    """ Get metadata for the given file

        :param str path: file name
        :param dict credDict: client credential dictionary

        :return: S_OK/S_ERROR, Value - dict of metadata
    """
    # First file metadata
    result = self._getFileMetadataFields(credDict)
    if not result['OK']:
      return result
    metaFields = result['Value']

    result = self.__getFileID(path)
    if not result['OK']:
      return result
    fileID = result['Value']

    metaDict = {}
    metaTypeDict = {}
    for meta in metaFields:
      req = "SELECT Value,FileID FROM FC_FileMeta_%s WHERE FileID=%d" % (meta, fileID)
      result = self.db._query(req)
      if not result['OK']:
        return result
      if result['Value']:
        metaDict[meta] = result['Value'][0][0]
      metaTypeDict[meta] = metaFields[meta]

    result = self.getFileMetaParameters(path, credDict)
    if result['OK']:
      metaDict.update(result['Value'])
      for meta in result['Value']:
        metaTypeDict[meta] = 'NonSearchable'

    result = S_OK(metaDict)
    result['MetadataType'] = metaTypeDict
    return result

  def __getFileMetaParameters(self, fileID, credDict):
    """ Get meta parameters for the given file

        :param int fileID: file ID
        :param dict credDict: client credential dictionary

        :return: S_OK/S_ERROR, Value - dict of meta parameters
    """

    req = "SELECT FileID,MetaKey,MetaValue from FC_FileMeta where FileID=%d " % fileID
    result = self.db._query(req)
    if not result['OK']:
      return result
    if not result['Value']:
      return S_OK({})
    metaDict = {}
    for fileID, key, value in result['Value']:
      if key in metaDict:
        if isinstance(metaDict[key], list):
          metaDict[key].append(value)
        else:
          metaDict[key] = [metaDict[key]].append(value)
      else:
        metaDict[key] = value

    return S_OK(metaDict)

  def getFileMetaParameters(self, path, credDict):
    """ Get meta parameters for the given file

        :param str path: file name
        :param dict credDict: client credential dictionary

        :return: S_OK/S_ERROR, Value - dict of meta parameters
    """

    result = self.__getFileID(path)
    if not result['OK']:
      return result
    fileID = result['Value']

    return self.__getFileMetaParameters(fileID, credDict)

  def __transformMetaParameterToData(self, metaName):
    """ Relocate the meta parameters of all the files to the corresponding
        indexed metadata table

        :param str metaName: meta parameter name

        :return: S_OK/S_ERROR
    """

    req = "SELECT FileID,MetaValue from FC_FileMeta WHERE MetaKey='%s'" % metaName
    result = self.db._query(req)
    if not result['OK']:
      return result
    if not result['Value']:
      return S_OK()

    insertValueList = []
    for fileID, meta in result['Value']:
      insertValueList.append("( %d,'%s' )" % (fileID, meta))

    req = "INSERT INTO FC_FileMeta_%s (FileID,Value) VALUES %s" % (metaName, ', '.join(insertValueList))
    result = self.db._update(req)
    if not result['OK']:
      return result

    req = "DELETE FROM FC_FileMeta WHERE MetaKey='%s'" % metaName
    result = self.db._update(req)
    return result

#########################################################################
#
#  Finding files by metadata
#
#########################################################################

  def __createMetaSelection(self, value):
    """ Create selection string for the given meta value to be used in the SQL query

        :param float,int,str value: metadata value

        :return: S_OK/S_ERROR, Value - list of SQL query selection elements
    """
    queryList = []
    if isinstance(value, float):
      queryList.append(('=', '%f' % value))
    elif isinstance(value, six.integer_types):
      queryList.append(('=', '%d' % value))
    elif isinstance(value, six.string_types):
      if value.lower() == 'any':
        queryList.append(('IS', 'NOT NULL'))
      elif value.lower() == 'missing':
        queryList.append(('IS', 'NULL'))
      elif value:
        result = self.db._escapeString(value)
        if not result['OK']:
          return result
        eValue = result['Value']
        if '*' in eValue or '?' in eValue:
          eValue = eValue.replace('*', '%%')
          eValue = eValue.replace('?', '_')
          queryList.append(('LIKE', eValue))
        else:
          queryList.append(('=', eValue))
      else:
        queryList.append(('', ''))
    elif isinstance(value, list):
      if not value:
        queryList.append(('', ''))
      else:
        result = self.db._escapeValues(value)
        if not result['OK']:
          return result
        query = '( $s )' % ', '.join(result['Value'])
        queryList.append(('IN', query))
    elif isinstance(value, dict):
      for operation, operand in value.items():

        # Prepare the escaped operand first
        if isinstance(operand, list):
          result = self.db._escapeValues(operand)
          if not result['OK']:
            return result
          escapedOperand = ', '.join(result['Value'])
        elif isinstance(operand, six.integer_types):
          escapedOperand = '%d' % operand
        elif isinstance(operand, float):
          escapedOperand = '%f' % operand
        else:
          result = self.db._escapeString(operand)
          if not result['OK']:
            return result
          escapedOperand = result['Value']

        # Treat the operations
        if operation in ['>', '<', '>=', '<=']:
          if isinstance(operand, list):
            return S_ERROR('Illegal query: list of values for comparison operation')
          else:
            queryList.append((operation, escapedOperand))
        elif operation == 'in' or operation == "=":
          if isinstance(operand, list):
            queryList.append(('IN', '( %s )' % escapedOperand))
          else:
            queryList.append(('=', escapedOperand))
        elif operation == 'nin' or operation == "!=":
          if isinstance(operand, list):
            queryList.append(('NOT IN', '( %s )' % escapedOperand))
          else:
            queryList.append(('!=', escapedOperand))

    return S_OK(queryList)

  def __buildSEQuery(self, storageElements):
    """ Return a tuple with table and condition to locate files in a given SE

        :param list storageElements: list of storage element names

        :return: S_OK/S_ERROR, Value - list of tuples (table name, SQL query elements)
    """
    if not storageElements:
      return S_OK([])

    seIDList = []
    for se in storageElements:
      seID = self.db.seNames.get(se, -1)
      if seID == -1:
        return S_ERROR('Unknown SE %s' % se)
      seIDList.append(seID)
    table = 'FC_Replicas'
    seString = intListToString(seIDList)
    query = '%%s.SEID IN ( %s )' % seString
    return S_OK([(table, query)])

  def __buildUserMetaQuery(self, userMetaDict):
    """ Return a list of tuples with tables and conditions to locate files for a given user Metadata

        :param dict userMetaDict: dictionary with user metadata

        :return: S_OK/S_ERROR, Value - list SQL query elements
    """
    if not userMetaDict:
      return S_OK([])
    resultList = []
    leftJoinTables = []
    for meta, value in userMetaDict.items():
      table = 'FC_FileMeta_%s' % meta

      result = self.__createMetaSelection(value)
      if not result['OK']:
        return result
      for operation, operand in result['Value']:
        resultList.append((table, '%%s.Value %s %s' % (operation, operand)))
        if operand == 'NULL':
          leftJoinTables.append(table)

    result = S_OK(resultList)
    result['LeftJoinTables'] = leftJoinTables
    return result

  def __buildStandardMetaQuery(self, standardMetaDict):
    """ Create SQL query elements for standard file metadata

        :param dict standardMetaDict: dictionary with standard file metadata

        :return: S_OK/S_ERROR, Value - list SQL query elements
    """

    table = 'FC_Files'
    queriesFiles = []
    queriesFileInfo = []
    for infield, invalue in standardMetaDict.items():
      value = invalue
      if infield in FILES_TABLE_METAKEYS:
        if infield == 'User':
          value = self.db.users.get(invalue, -1)
          if value == '-1':
            return S_ERROR('Unknown user %s' % invalue)
        elif infield == 'Group':
          value = self.db.groups.get(invalue, -1)
          if value == '-1':
            return S_ERROR('Unknown group %s' % invalue)

        table = 'FC_Files'
        tableIndex = 'F'
        field = FILES_TABLE_METAKEYS[infield]
        result = self.__createMetaSelection(value)
        if not result['OK']:
          return result
        for operation, operand in result['Value']:
          queriesFiles.append('%s.%s %s %s' % (tableIndex, field, operation, operand))
      elif infield in FILEINFO_TABLE_METAKEYS:
        table = 'FC_FileInfo'
        tableIndex = 'FI'
        field = FILEINFO_TABLE_METAKEYS[infield]
        result = self.__createMetaSelection(value)
        if not result['OK']:
          return result
        for operation, operand in result['Value']:
          queriesFileInfo.append('%s.%s %s %s' % (tableIndex, field, operation, operand))
      else:
        return S_ERROR('Illegal standard meta key %s' % infield)

    resultList = []
    if queriesFiles:
      query = ' AND '.join(queriesFiles)
      resultList.append(('FC_Files', query))
    if queriesFileInfo:
      query = ' AND '.join(queriesFileInfo)
      resultList.append(('FC_FileInfo', query))

    return S_OK(resultList)

  def __findFilesByMetadata(self, metaDict, dirList, credDict):
    """ Find a list of file IDs meeting the metaDict requirements and belonging
        to directories in dirList

        :param dict metaDict: dictionary with the file metadata
        :param list dirList: list of directories to look into

        :return: S_OK/S_ERROR, Value - list of IDs of found files
    """
    # 1.- classify Metadata keys
    storageElements = None
    standardMetaDict = {}
    userMetaDict = {}
    leftJoinTables = []
    for meta, value in metaDict.items():
      if meta == "SE":
        if isinstance(value, dict):
          storageElements = value.get('in', [])
        else:
          storageElements = [value]
      elif meta in FILE_STANDARD_METAKEYS:
        standardMetaDict[meta] = value
      else:
        userMetaDict[meta] = value

    tablesAndConditions = []
    leftJoinTables = []
    # 2.- standard search
    if standardMetaDict:
      result = self.__buildStandardMetaQuery(standardMetaDict)
      if not result['OK']:
        return result
      tablesAndConditions.extend(result['Value'])
    # 3.- user search
    if userMetaDict:
      result = self.__buildUserMetaQuery(userMetaDict)
      if not result['OK']:
        return result
      tablesAndConditions.extend(result['Value'])
      leftJoinTables = result['LeftJoinTables']
    # 4.- SE constraint
    if storageElements:
      result = self.__buildSEQuery(storageElements)
      if not result['OK']:
        return result
      tablesAndConditions.extend(result['Value'])

    query = 'SELECT F.FileID FROM FC_Files F '
    conditions = []
    tables = []

    if dirList:
      dirString = intListToString(dirList)
      conditions.append("F.DirID in (%s)" % dirString)

    counter = 0
    for table, condition in tablesAndConditions:
      if table == 'FC_FileInfo':
        query += 'INNER JOIN FC_FileInfo FI USING( FileID ) '
        condition = condition.replace('%%', '%')
      elif table == 'FC_Files':
        condition = condition.replace('%%', '%')
      else:
        counter += 1
        if table in leftJoinTables:
          tables.append('LEFT JOIN %s M%d USING( FileID )' % (table, counter))
        else:
          tables.append('INNER JOIN %s M%d USING( FileID )' % (table, counter))
        table = 'M%d' % counter
        condition = condition % table
      conditions.append(condition)

    query += ' '.join(tables)
    if conditions:
      query += ' WHERE %s' % ' AND '.join(conditions)

    result = self.db._query(query)
    if not result['OK']:
      return result
    if not result['Value']:
      return S_OK([])

#     fileList = [ row[0] for row in result['Value' ] ]
    fileList = []
    for row in result['Value']:
      fileID = row[0]
      fileList.append(fileID)

    return S_OK(fileList)

  @queryTime
  def findFilesByMetadata(self, metaDict, path, credDict):
    """ Find Files satisfying the given metadata

        :param dict metaDict: dictionary with the metaquery parameters
        :param str path: Path to search into
        :param dict credDict: Dictionary with the user credentials

        :return: S_OK/S_ERROR, Value ID:LFN dictionary of selected files
    """
    if not path:
      path = '/'

    # 1.- Get Directories matching the metadata query
    result = self.db.dmeta.findDirIDsByMetadata(metaDict, path, credDict)
    if not result['OK']:
      return result
    dirList = result['Value']
    dirFlag = result['Selection']

    # 2.- Get known file metadata fields
#     fileMetaDict = {}
    result = self._getFileMetadataFields(credDict)
    if not result['OK']:
      return result
    fileMetaKeys = list(result['Value']) + list(FILE_STANDARD_METAKEYS)
    fileMetaDict = dict(item for item in metaDict.items() if item[0] in fileMetaKeys)

    fileList = []
    idLfnDict = {}

    if dirFlag != 'None':
      # None means that no Directory satisfies the given query, thus the search is empty
      if dirFlag == 'All':
        # All means that there is no Directory level metadata in query, full name space is considered
        dirList = []

      if fileMetaDict:
        # 3.- Do search in File Metadata
        result = self.__findFilesByMetadata(fileMetaDict, dirList, credDict)
        if not result['OK']:
          return result
        fileList = result['Value']
      elif dirList:
        # 4.- if not File Metadata, return the list of files in given directories
        result = self.db.dtree.getFileLFNsInDirectoryByDirectory(dirList, credDict)
        if not result['OK']:
          return result
        return S_OK(result['Value']['IDLFNDict'])
      else:
        # if there is no File Metadata and no Dir Metadata, return an empty list
        idLfnDict = {}

    if fileList:
      # 5.- get the LFN
      result = self.db.fileManager._getFileLFNs(fileList)
      if not result['OK']:
        return result
      idLfnDict = result['Value']['Successful']

    return S_OK(idLfnDict)
