""" TaskQueueDB class is a front-end to the task queues db
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

__RCSID__ = "$Id"

import six
import random
import string

from DIRAC import gConfig, S_OK, S_ERROR
from DIRAC.Core.Base.DB import DB
from DIRAC.Core.Utilities import List
from DIRAC.Core.Utilities.PrettyPrint import printDict
from DIRAC.Core.Utilities.DictCache import DictCache
from DIRAC.Core.Security import Properties
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
from DIRAC.ConfigurationSystem.Client.Helpers import Registry
from DIRAC.WorkloadManagementSystem.private.SharesCorrector import SharesCorrector

DEFAULT_GROUP_SHARE = 1000
TQ_MIN_SHARE = 0.001

# For checks at insertion time, and not only
singleValueDefFields = ('OwnerDN', 'OwnerGroup', 'Setup', 'CPUTime')
multiValueDefFields = ('Sites', 'GridCEs', 'BannedSites',
                       'Platforms', 'SubmitPools', 'JobTypes', 'Tags')

# Used for matching
multiValueMatchFields = ('GridCE', 'Site', 'Platform',
                         'SubmitPool', 'JobType', 'Tag')
bannedJobMatchFields = ('Site', )
mandatoryMatchFields = ('Setup', 'CPUTime')
priorityIgnoredFields = ('Sites', 'BannedSites')


class TaskQueueDB(DB):
  """ MySQL DB of "Task Queues"
  """

  def __init__(self):
    random.seed()
    DB.__init__(self, 'TaskQueueDB', 'WorkloadManagement/TaskQueueDB')
    self.__maxJobsInTQ = 5000
    self.__defaultCPUSegments = [6 * 60,
                                 30 * 60,
                                 1 * 3600,
                                 6 * 3600,
                                 12 * 3600,
                                 1 * 86400,
                                 2 * 86400,
                                 3 * 86400,
                                 4 * 86400,
                                 6 * 86400,
                                 8 * 86400,
                                 10 * 86400,
                                 int(12.5 * 86400)]
    self.__maxMatchRetry = 3
    self.__jobPriorityBoundaries = (0.001, 10)
    self.__groupShares = {}
    self.__deleteTQWithDelay = DictCache(self.__deleteTQIfEmpty)
    self.__opsHelper = Operations()
    self.__ensureInsertionIsSingle = False
    self.__sharesCorrector = SharesCorrector(self.__opsHelper)
    result = self.__initializeDB()
    if not result['OK']:
      raise Exception("Can't create tables: %s" % result['Message'])

  def enableAllTaskQueues(self):
    """ Enable all Task queues
    """
    return self.updateFields("tq_TaskQueues", updateDict={"Enabled": "1"})

  def findOrphanJobs(self):
    """ Find jobs that are not in any task queue
    """
    result = self._query("select JobID from tq_Jobs WHERE TQId not in (SELECT TQId from tq_TaskQueues)")
    if not result['OK']:
      return result
    return S_OK([row[0] for row in result['Value']])

  def isSharesCorrectionEnabled(self):
    return self.__getCSOption("EnableSharesCorrection", False)

  def __getCSOption(self, optionName, defValue):
    return self.__opsHelper.getValue("JobScheduling/%s" % optionName, defValue)

  def getValidPilotTypes(self):
    return self.__getCSOption("AllPilotTypes", ['private'])

  def __initializeDB(self):
    """
    Create the tables
    """
    result = self._query("show tables")
    if not result['OK']:
      return result

    tablesInDB = [t[0] for t in result['Value']]
    tablesToCreate = {}
    self.__tablesDesc = {}

    self.__tablesDesc['tq_TaskQueues'] = {'Fields': {'TQId': 'INTEGER(11) UNSIGNED AUTO_INCREMENT NOT NULL',
                                                     'OwnerDN': 'VARCHAR(255) NOT NULL',
                                                     'OwnerGroup': 'VARCHAR(32) NOT NULL',
                                                     'Setup': 'VARCHAR(32) NOT NULL',
                                                     'CPUTime': 'BIGINT(20) UNSIGNED NOT NULL',
                                                     'Priority': 'FLOAT NOT NULL',
                                                     'Enabled': 'TINYINT(1) NOT NULL DEFAULT 0'
                                                     },
                                          'PrimaryKey': 'TQId',
                                          'Indexes': {'TQOwner': ['OwnerDN', 'OwnerGroup',
                                                                  'Setup', 'CPUTime']
                                                      }
                                          }

    self.__tablesDesc['tq_Jobs'] = {'Fields': {'TQId': 'INTEGER(11) UNSIGNED NOT NULL',
                                               'JobId': 'INTEGER(11) UNSIGNED NOT NULL',
                                               'Priority': 'INTEGER UNSIGNED NOT NULL',
                                               'RealPriority': 'FLOAT NOT NULL'
                                               },
                                    'PrimaryKey': 'JobId',
                                    'Indexes': {'TaskIndex': ['TQId']},
                                    'ForeignKeys': {'TQId': 'tq_TaskQueues.TQId'}
                                    }

    for multiField in multiValueDefFields:
      tableName = 'tq_TQTo%s' % multiField
      self.__tablesDesc[tableName] = {'Fields': {'TQId': 'INTEGER(11) UNSIGNED NOT NULL',
                                                 'Value': 'VARCHAR(64) NOT NULL'
                                                 },
                                      'PrimaryKey': ['TQId', 'Value'],
                                      'Indexes': {'TaskIndex': ['TQId'], '%sIndex' % multiField: ['Value']},
                                      'ForeignKeys': {'TQId': 'tq_TaskQueues.TQId'}
                                      }

    for tableName in self.__tablesDesc:
      if tableName not in tablesInDB:
        tablesToCreate[tableName] = self.__tablesDesc[tableName]

    return self._createTables(tablesToCreate)

  def getGroupsInTQs(self):
    cmdSQL = "SELECT DISTINCT( OwnerGroup ) FROM `tq_TaskQueues`"
    result = self._query(cmdSQL)
    if not result['OK']:
      return result
    return S_OK([row[0] for row in result['Value']])

  def fitCPUTimeToSegments(self, cpuTime):
    """
    Fit the CPU time to the valid segments
    """
    maxCPUSegments = self.__getCSOption("taskQueueCPUTimeIntervals", self.__defaultCPUSegments)
    try:
      maxCPUSegments = [int(seg) for seg in maxCPUSegments]
      # Check segments in the CS
      last = 0
      for cpuS in maxCPUSegments:
        if cpuS <= last:
          maxCPUSegments = self.__defaultCPUSegments
          break
        last = cpuS
    except BaseException:
      maxCPUSegments = self.__defaultCPUSegments
    # Map to a segment
    for cpuSegment in maxCPUSegments:
      if cpuTime <= cpuSegment:
        return cpuSegment
    return maxCPUSegments[-1]

  def _checkTaskQueueDefinition(self, tqDefDict):
    """
    Check a task queue definition dict is valid
    """

    for field in singleValueDefFields:
      if field not in tqDefDict:
        return S_ERROR("Missing mandatory field '%s' in task queue definition" % field)
      if field in ["CPUTime"]:
        if not isinstance(tqDefDict[field], six.integer_types):
          return S_ERROR("Mandatory field %s value type is not valid: %s" % (field, type(tqDefDict[field])))
      else:
        if not isinstance(tqDefDict[field], six.string_types):
          return S_ERROR("Mandatory field %s value type is not valid: %s" % (field, type(tqDefDict[field])))
        result = self._escapeString(tqDefDict[field])
        if not result['OK']:
          return result
        tqDefDict[field] = result['Value']
    for field in multiValueDefFields:
      if field not in tqDefDict:
        continue
      if not isinstance(tqDefDict[field], (list, tuple)):
        return S_ERROR("Multi value field %s value type is not valid: %s" % (field, type(tqDefDict[field])))
      result = self._escapeValues(tqDefDict[field])
      if not result['OK']:
        return result
      tqDefDict[field] = result['Value']

    return S_OK(tqDefDict)

  def _checkMatchDefinition(self, tqMatchDict):
    """
    Check a task queue match dict is valid
    """
    def travelAndCheckType(value, validTypes, escapeValues=True):
      if isinstance(value, (list, tuple)):
        for subValue in value:
          if not isinstance(subValue, validTypes):
            return S_ERROR("List contained type %s is not valid -> %s" % (type(subValue), validTypes))
        if escapeValues:
          return self._escapeValues(value)
        return S_OK(value)
      else:
        if not isinstance(value, validTypes):
          return S_ERROR("Type %s is not valid -> %s" % (type(value), validTypes))
        if escapeValues:
          return self._escapeString(value)
        return S_OK(value)

    for field in singleValueDefFields:
      if field not in tqMatchDict:
        if field in mandatoryMatchFields:
          return S_ERROR("Missing mandatory field '%s' in match request definition" % field)
        continue
      fieldValue = tqMatchDict[field]
      if field in ["CPUTime"]:
        result = travelAndCheckType(fieldValue, six.integer_types, escapeValues=False)
      else:
        result = travelAndCheckType(fieldValue, six.string_types)
      if not result['OK']:
        return S_ERROR("Match definition field %s failed : %s" % (field, result['Message']))
      tqMatchDict[field] = result['Value']
    # Check multivalue
    for multiField in multiValueMatchFields:
      for field in (multiField, "Banned%s" % multiField, "Required%s" % multiField):
        if field in tqMatchDict:
          fieldValue = tqMatchDict[field]
          result = travelAndCheckType(fieldValue, six.string_types)
          if not result['OK']:
            return S_ERROR("Match definition field %s failed : %s" % (field, result['Message']))
          tqMatchDict[field] = result['Value']

    return S_OK(tqMatchDict)

  def __createTaskQueue(self, tqDefDict, priority=1, connObj=False):
    """
    Create a task queue
      :returns: S_OK( tqId ) / S_ERROR
    """
    if not connObj:
      result = self._getConnection()
      if not result['OK']:
        return S_ERROR("Can't create task queue: %s" % result['Message'])
      connObj = result['Value']
    tqDefDict['CPUTime'] = self.fitCPUTimeToSegments(tqDefDict['CPUTime'])
    sqlSingleFields = ['TQId', 'Priority']
    sqlValues = ["0", str(priority)]
    for field in singleValueDefFields:
      sqlSingleFields.append(field)
      sqlValues.append(tqDefDict[field])
    # Insert the TQ Disabled
    sqlSingleFields.append("Enabled")
    sqlValues.append("0")
    cmd = "INSERT INTO tq_TaskQueues ( %s ) VALUES ( %s )" % (
        ", ".join(sqlSingleFields), ", ".join([str(v) for v in sqlValues]))
    result = self._update(cmd, conn=connObj)
    if not result['OK']:
      self.log.error("Can't insert TQ in DB", result['Value'])
      return result
    if 'lastRowId' in result:
      tqId = result['lastRowId']
    else:
      result = self._query("SELECT LAST_INSERT_ID()", conn=connObj)
      if not result['OK']:
        self.cleanOrphanedTaskQueues(connObj=connObj)
        return S_ERROR("Can't determine task queue id after insertion")
      tqId = result['Value'][0][0]
    for field in multiValueDefFields:
      if field not in tqDefDict:
        continue
      values = List.uniqueElements([value for value in tqDefDict[field] if value.strip()])
      if not values:
        continue
      cmd = "INSERT INTO `tq_TQTo%s` ( TQId, Value ) VALUES " % field
      cmd += ", ".join(["( %s, %s )" % (tqId, str(value)) for value in values])
      result = self._update(cmd, conn=connObj)
      if not result['OK']:
        self.log.error("Failed to insert condition",
                       "%s : %s" % field, result['Message'])
        self.cleanOrphanedTaskQueues(connObj=connObj)
        return S_ERROR("Can't insert values %s for field %s: %s" % (str(values), field, result['Message']))
    self.log.info("Created TQ", tqId)
    return S_OK(tqId)

  def cleanOrphanedTaskQueues(self, connObj=False):
    """
    Delete all empty task queues
    """
    self.log.info("Cleaning orphaned TQs")
    sq = "SELECT TQId FROM `tq_TaskQueues` WHERE Enabled >= 1 AND TQId not in ( SELECT DISTINCT TQId from `tq_Jobs` )"
    result = self._query(sq, conn=connObj)
    if not result['OK']:
      return result
    orphanedTQs = result['Value']
    if not orphanedTQs:
      return S_OK()
    orphanedTQs = [str(otq[0]) for otq in orphanedTQs]

    for mvField in multiValueDefFields:
      result = self._update(
          "DELETE FROM `tq_TQTo%s` WHERE TQId in ( %s )" % (mvField, ','.join(orphanedTQs)), conn=connObj)
      if not result['OK']:
        return result

    result = self._update(
        "DELETE FROM `tq_TaskQueues` WHERE TQId in ( %s )" % ','.join(orphanedTQs), conn=connObj)
    if not result['OK']:
      return result
    return S_OK()

  def __setTaskQueueEnabled(self, tqId, enabled=True, connObj=False):
    if enabled:
      enabled = "+ 1"
    else:
      enabled = "- 1"
    upSQL = "UPDATE `tq_TaskQueues` SET Enabled = Enabled %s WHERE TQId=%d" % (enabled, tqId)
    result = self._update(upSQL, conn=connObj)
    if not result['OK']:
      self.log.error("Error setting TQ state", "TQ %s State %s: %s" % (tqId, enabled, result['Message']))
      return result
    updated = result['Value'] > 0
    if updated:
      self.log.verbose("Set enabled for TQ",
                       "(%s for TQ %s)" % (enabled, tqId))
    return S_OK(updated)

  def __hackJobPriority(self, jobPriority):
    jobPriority = min(max(int(jobPriority), self.__jobPriorityBoundaries[0]), self.__jobPriorityBoundaries[1])
    if jobPriority == self.__jobPriorityBoundaries[0]:
      return 10 ** (-5)
    if jobPriority == self.__jobPriorityBoundaries[1]:
      return 10 ** 6
    return jobPriority

  def insertJob(self, jobId, tqDefDict, jobPriority, skipTQDefCheck=False):
    """ Insert a job in a task queue (creating one if it doesn't exit)

        :param int jobId: job ID
        :param dict tqDefDict: dict for TQ definition
        :param int jobPriority: integer that defines the job priority

        :returns: S_OK() / S_ERROR
    """
    try:
      int(jobId)
    except ValueError:
      return S_ERROR("JobId is not a number!")
    retVal = self._getConnection()
    if not retVal['OK']:
      return S_ERROR("Can't insert job: %s" % retVal['Message'])
    connObj = retVal['Value']
    if not skipTQDefCheck:
      tqDefDict = dict(tqDefDict)
      retVal = self._checkTaskQueueDefinition(tqDefDict)
      if not retVal['OK']:
        self.log.error("TQ definition check failed", retVal['Message'])
        return retVal
      tqDefDict = retVal['Value']
    tqDefDict['CPUTime'] = self.fitCPUTimeToSegments(tqDefDict['CPUTime'])
    self.log.info("Inserting job with requirements",
                  "(%s : %s)" % (jobId, printDict(tqDefDict)))
    retVal = self.__findAndDisableTaskQueue(tqDefDict, skipDefinitionCheck=True, connObj=connObj)
    if not retVal['OK']:
      return retVal
    tqInfo = retVal['Value']
    newTQ = False
    if not tqInfo['found']:
      self.log.info("Creating a TQ for job", jobId)
      retVal = self.__createTaskQueue(tqDefDict, 1, connObj=connObj)
      if not retVal['OK']:
        return retVal
      tqId = retVal['Value']
      newTQ = True
    else:
      tqId = tqInfo['tqId']
      self.log.info("Found TQ for job requirements",
                    "(%s : %s)" % (tqId, jobId))
    try:
      result = self.__insertJobInTaskQueue(jobId, tqId, int(jobPriority), checkTQExists=False, connObj=connObj)
      if not result['OK']:
        self.log.error("Error inserting job in TQ", "Job %s TQ %s: %s" % (jobId, tqId, result['Message']))
        return result
      if newTQ:
        self.recalculateTQSharesForEntity(tqDefDict['OwnerDN'], tqDefDict['OwnerGroup'], connObj=connObj)
    finally:
      self.__setTaskQueueEnabled(tqId, True)
    return S_OK()

  def __insertJobInTaskQueue(self, jobId, tqId, jobPriority, checkTQExists=True, connObj=False):
    """ Insert a job in a given task queue

        :param int jobId: job ID
        :param dict tqDefDict: dict for TQ definition
        :param int jobPriority: integer that defines the job priority

        :returns: S_OK() / S_ERROR
    """
    self.log.info("Inserting job in TQ with priority",
                  "(%s : %s : %s)" % (jobId, tqId, jobPriority))
    if not connObj:
      result = self._getConnection()
      if not result['OK']:
        return S_ERROR("Can't insert job: %s" % result['Message'])
      connObj = result['Value']
    if checkTQExists:
      result = self._query("SELECT tqId FROM `tq_TaskQueues` WHERE TQId = %s" % tqId, conn=connObj)
      if not result['OK'] or not result['Value']:
        return S_OK("Can't find task queue with id %s: %s" % (tqId, result['Message']))
    hackedPriority = self.__hackJobPriority(jobPriority)
    result = self._update("INSERT INTO tq_Jobs ( TQId, JobId, Priority, RealPriority ) \
                            VALUES ( %s, %s, %s, %f ) ON DUPLICATE KEY UPDATE TQId = %s, \
                            Priority = %s, RealPriority = %f" % (tqId, jobId, jobPriority, hackedPriority,
                                                                 tqId, jobPriority, hackedPriority),
                          conn=connObj)
    if not result['OK']:
      return result
    return S_OK()

  def __generateTQFindSQL(self, tqDefDict, skipDefinitionCheck=False):
    """
        Generate the SQL to find a task queue that has exactly the given requirements

        :param dict tqDefDict: dict for TQ definition
        :returns: S_OK() / S_ERROR
    """
    if not skipDefinitionCheck:
      tqDefDict = dict(tqDefDict)
      result = self._checkTaskQueueDefinition(tqDefDict)
      if not result['OK']:
        return result
      tqDefDict = result['Value']

    sqlCondList = []
    for field in singleValueDefFields:
      sqlCondList.append("`tq_TaskQueues`.%s = %s" % (field, tqDefDict[field]))
    # MAGIC SUBQUERIES TO ENSURE STRICT MATCH
    for field in multiValueDefFields:
      tableName = '`tq_TQTo%s`' % field
      if field in tqDefDict and tqDefDict[field]:
        firstQuery = "SELECT COUNT(%s.Value) \
                      FROM %s \
                      WHERE %s.TQId = `tq_TaskQueues`.TQId" % (tableName, tableName, tableName)
        grouping = "GROUP BY %s.TQId" % tableName
        valuesList = List.uniqueElements([value.strip() for value in tqDefDict[field] if value.strip()])
        numValues = len(valuesList)
        secondQuery = "%s AND %s.Value in (%s)" % (firstQuery, tableName,
                                                   ",".join(["%s" % str(value) for value in valuesList]))
        sqlCondList.append("%s = (%s %s)" % (numValues, firstQuery, grouping))
        sqlCondList.append("%s = (%s %s)" % (numValues, secondQuery, grouping))
      else:
        sqlCondList.append("`tq_TaskQueues`.TQId not in ( SELECT DISTINCT %s.TQId from %s )" % (tableName, tableName))
    # END MAGIC: That was easy ;)
    return S_OK(" AND ".join(sqlCondList))

  def __findAndDisableTaskQueue(self, tqDefDict, skipDefinitionCheck=False, retries=10, connObj=False):
    """ Disable and find TQ

        :param dict tqDefDict: dict for TQ definition
        :returns: S_OK() / S_ERROR
    """
    for _ in range(retries):
      result = self.__findSmallestTaskQueue(tqDefDict, skipDefinitionCheck=skipDefinitionCheck, connObj=connObj)
      if not result['OK']:
        return result
      data = result['Value']
      if not data['found']:
        return result
      if data['enabled'] < 1:
        self.log.debug("TaskQueue {tqId} seems to be already disabled ({enabled})".format(**data))
      result = self.__setTaskQueueEnabled(data['tqId'], False)
      if result['OK']:
        return S_OK(data)
    return S_ERROR("Could not disable TQ")

  def __findSmallestTaskQueue(self, tqDefDict, skipDefinitionCheck=False, connObj=False):
    """
        Find a task queue that has at least the given requirements

        :param dict tqDefDict: dict for TQ definition
        :returns: S_OK() / S_ERROR
    """
    result = self.__generateTQFindSQL(tqDefDict, skipDefinitionCheck=skipDefinitionCheck)
    if not result['OK']:
      return result

    sqlCmd = "SELECT COUNT( `tq_Jobs`.JobID ), `tq_TaskQueues`.TQId, `tq_TaskQueues`.Enabled \
FROM `tq_TaskQueues`, `tq_Jobs`"
    sqlCmd = "%s WHERE `tq_TaskQueues`.TQId = `tq_Jobs`.TQId AND %s GROUP BY `tq_Jobs`.TQId \
ORDER BY COUNT( `tq_Jobs`.JobID ) ASC" % (sqlCmd, result['Value'])
    result = self._query(sqlCmd, conn=connObj)
    if not result['OK']:
      self.log.error("Can't find task queue", result['Message'])
      return result
    data = result['Value']
    if not data or data[0][0] >= self.__maxJobsInTQ:
      return S_OK({'found': False})
    return S_OK({'found': True, 'tqId': data[0][1], 'enabled': data[0][2], 'jobs': data[0][0]})

  def matchAndGetJob(self, tqMatchDict, numJobsPerTry=50, numQueuesPerTry=10, negativeCond=None):
    """ Match a job based on requirements

        :param dict tqDefDict: dict for TQ definition
        :returns: S_OK() / S_ERROR
    """
    if negativeCond is None:
      negativeCond = {}
    # Make a copy to avoid modification of original if escaping needs to be done
    tqMatchDict = dict(tqMatchDict)
    retVal = self._checkMatchDefinition(tqMatchDict)
    if not retVal['OK']:
      self.log.error("TQ match request check failed", retVal['Message'])
      return retVal
    retVal = self._getConnection()
    if not retVal['OK']:
      return S_ERROR("Can't connect to DB: %s" % retVal['Message'])
    connObj = retVal['Value']
    preJobSQL = "SELECT `tq_Jobs`.JobId, `tq_Jobs`.TQId \
FROM `tq_Jobs` WHERE `tq_Jobs`.TQId = %s AND `tq_Jobs`.Priority = %s"
    prioSQL = "SELECT `tq_Jobs`.Priority FROM `tq_Jobs` \
WHERE `tq_Jobs`.TQId = %s ORDER BY RAND() / `tq_Jobs`.RealPriority ASC LIMIT 1"
    postJobSQL = " ORDER BY `tq_Jobs`.JobId ASC LIMIT %s" % numJobsPerTry
    for _ in range(self.__maxMatchRetry):
      noJobsFound = False
      if 'JobID' in tqMatchDict:
        # A certain JobID is required by the resource, so all TQ are to be considered
        retVal = self.matchAndGetTaskQueue(tqMatchDict,
                                           numQueuesToGet=0,
                                           skipMatchDictDef=True,
                                           connObj=connObj)
        preJobSQL = "%s AND `tq_Jobs`.JobId = %s " % (preJobSQL, tqMatchDict['JobID'])
      else:
        retVal = self.matchAndGetTaskQueue(tqMatchDict,
                                           numQueuesToGet=numQueuesPerTry,
                                           skipMatchDictDef=True,
                                           negativeCond=negativeCond,
                                           connObj=connObj)
      if not retVal['OK']:
        return retVal
      tqList = retVal['Value']
      if not tqList:
        self.log.info("No TQ matches requirements")
        return S_OK({'matchFound': False, 'tqMatch': tqMatchDict})
      for tqId, tqOwnerDN, tqOwnerGroup in tqList:
        self.log.verbose("Trying to extract jobs from TQ", tqId)
        retVal = self._query(prioSQL % tqId, conn=connObj)
        if not retVal['OK']:
          return S_ERROR("Can't retrieve winning priority for matching job: %s" % retVal['Message'])
        if not retVal['Value']:
          noJobsFound = True
          continue
        prio = retVal['Value'][0][0]
        retVal = self._query("%s %s" % (preJobSQL % (tqId, prio), postJobSQL), conn=connObj)
        if not retVal['OK']:
          return S_ERROR("Can't begin transaction for matching job: %s" % retVal['Message'])
        jobTQList = [(row[0], row[1]) for row in retVal['Value']]
        if not jobTQList:
          self.log.info("Task queue seems to be empty, triggering a cleaning of", tqId)
          self.__deleteTQWithDelay.add(tqId, 300, (tqId, tqOwnerDN, tqOwnerGroup))
        while jobTQList:
          jobId, tqId = jobTQList.pop(random.randint(0, len(jobTQList) - 1))
          self.log.verbose("Trying to extract job from TQ",
                           "%s : %s" % (jobId, tqId))
          retVal = self.deleteJob(jobId, connObj=connObj)
          if not retVal['OK']:
            msgFix = "Could not take job"
            msgVar = " %s out from the TQ %s: %s" % (jobId, tqId, retVal['Message'])
            self.log.error(msgFix, msgVar)
            return S_ERROR(msgFix + msgVar)
          if retVal['Value']:
            self.log.info("Extracted job with prio from TQ",
                          "(%s : %s : %s)" % (jobId, prio, tqId))
            return S_OK({'matchFound': True, 'jobId': jobId, 'taskQueueId': tqId, 'tqMatch': tqMatchDict})
        self.log.info("No jobs could be extracted from TQ", tqId)
    if noJobsFound:
      return S_OK({'matchFound': False, 'tqMatch': tqMatchDict})

    self.log.info("Could not find a match after %s match retries" % self.__maxMatchRetry)
    return S_ERROR("Could not find a match after %s match retries" % self.__maxMatchRetry)

  def matchAndGetTaskQueue(self, tqMatchDict, numQueuesToGet=1, skipMatchDictDef=False,
                           negativeCond=None, connObj=False):
    """ Get a queue that matches the requirements
    """
    if negativeCond is None:
      negativeCond = {}
    # Make a copy to avoid modification of original if escaping needs to be done
    tqMatchDict = dict(tqMatchDict)
    if not skipMatchDictDef:
      retVal = self._checkMatchDefinition(tqMatchDict)
      if not retVal['OK']:
        return retVal
    retVal = self.__generateTQMatchSQL(tqMatchDict, numQueuesToGet=numQueuesToGet, negativeCond=negativeCond)
    if not retVal['OK']:
      return retVal
    matchSQL = retVal['Value']
    retVal = self._query(matchSQL, conn=connObj)
    if not retVal['OK']:
      return retVal
    return S_OK([(row[0], row[1], row[2]) for row in retVal['Value']])

  @staticmethod
  def __generateSQLSubCond(sqlString, value, boolOp='OR'):
    if not isinstance(value, (list, tuple)):
      return sqlString % str(value).strip()
    sqlORList = []
    for v in value:
      sqlORList.append(sqlString % str(v).strip())
    return "( %s )" % (" %s " % boolOp).join(sqlORList)

  def __generateNotSQL(self, negativeCond):
    """ Generate negative conditions
        Can be a list of dicts or a dict:
         - list of dicts will be  OR of conditional dicts
         - dicts will be normal conditional dict ( kay1 in ( v1, v2, ... ) AND key2 in ( v3, v4, ... ) )
    """
    if isinstance(negativeCond, (list, tuple)):
      sqlCond = []
      for cD in negativeCond:
        sqlCond.append(self.__generateNotDictSQL(cD))
      return " ( %s )" % " OR  ".join(sqlCond)
    elif isinstance(negativeCond, dict):
      return self.__generateNotDictSQL(negativeCond)
    raise RuntimeError("negativeCond has to be either a list or a dict or a tuple, and it's %s" % type(negativeCond))

  def __generateNotDictSQL(self, negativeCond):
    """ Generate the negative sql condition from a standard condition dict
        not ( cond1 and cond2 ) = ( not cond1 or not cond 2 )
        For instance: { 'Site': 'S1', 'JobType': [ 'T1', 'T2' ] }
          ( not 'S1' in Sites or ( not 'T1' in JobType and not 'T2' in JobType ) )
          S2 T1 -> not False or ( not True and not False ) -> True or ... -> True -> Eligible
          S1 T3 -> not True or ( not False and not False ) -> False or (True and True ) -> True -> Eligible
          S1 T1 -> not True or ( not True and not False ) -> False or ( False and True ) -> False -> Nop
    """
    condList = []
    for field in negativeCond:
      if field in multiValueMatchFields:
        fullTableN = '`tq_TQTo%ss`' % field
        valList = negativeCond[field]
        if not isinstance(valList, (list, tuple)):
          valList = (valList, )
        subList = []
        for value in valList:
          value = self._escapeString(value)['Value']
          sql = "%s NOT IN ( SELECT %s.Value FROM %s WHERE %s.TQId = tq.TQId )" % (value,
                                                                                   fullTableN,
                                                                                   fullTableN,
                                                                                   fullTableN)
          subList.append(sql)
        condList.append("( %s )" % " AND ".join(subList))
      elif field in singleValueDefFields:
        for value in negativeCond[field]:
          value = self._escapeString(value)['Value']
          sql = "%s != tq.%s " % (value, field)
          condList.append(sql)
    return "( %s )" % " OR ".join(condList)

  @staticmethod
  def __generateTablesName(sqlTables, field):
    fullTableName = 'tq_TQTo%ss' % field
    if fullTableName not in sqlTables:
      tableN = field.lower()
      sqlTables[fullTableName] = tableN
      return tableN, "`%s`" % fullTableName,
    return sqlTables[fullTableName], "`%s`" % fullTableName

  def __generateTQMatchSQL(self, tqMatchDict, numQueuesToGet=1, negativeCond=None):
    """
    Generate the SQL needed to match a task queue
    """
    self.log.debug(tqMatchDict)

    if negativeCond is None:
      negativeCond = {}
    # Only enabled TQs
    sqlCondList = []
    sqlTables = {"tq_TaskQueues": "tq"}
    # If OwnerDN and OwnerGroup are defined only use those combinations that make sense
    if 'OwnerDN' in tqMatchDict and 'OwnerGroup' in tqMatchDict:
      groups = tqMatchDict['OwnerGroup']
      if not isinstance(groups, (list, tuple)):
        groups = [groups]
      dns = tqMatchDict['OwnerDN']
      if not isinstance(dns, (list, tuple)):
        dns = [dns]
      ownerConds = []
      for group in groups:
        if Properties.JOB_SHARING in Registry.getPropertiesForGroup(group.replace('"', "")):
          ownerConds.append("tq.OwnerGroup = %s" % group)
        else:
          for dn in dns:
            ownerConds.append("( tq.OwnerDN = %s AND tq.OwnerGroup = %s )" % (dn, group))
      sqlCondList.append(" OR ".join(ownerConds))
    else:
      # If not both are defined, just add the ones that are defined
      for field in ('OwnerGroup', 'OwnerDN'):
        if field in tqMatchDict:
          sqlCondList.append(self.__generateSQLSubCond("tq.%s = %%s" % field,
                                                       tqMatchDict[field]))
    # Type of single value conditions
    for field in ('CPUTime', 'Setup'):
      if field in tqMatchDict:
        if field == 'CPUTime':
          sqlCondList.append(self.__generateSQLSubCond("tq.%s <= %%s" % field, tqMatchDict[field]))
        else:
          sqlCondList.append(self.__generateSQLSubCond("tq.%s = %%s" % field, tqMatchDict[field]))

    tag_fv = []

    # Match multi value fields
    for field in multiValueMatchFields:
      self.log.debug("Evaluating field %s" % field)
      # It has to be %ss , with an 's' at the end because the columns names
      # are plural and match options are singular

      # Just treating the (not so) special case of no Tag, No RequiredTag
      if 'Tag' not in tqMatchDict and 'RequiredTag' not in tqMatchDict:
        tqMatchDict['Tag'] = []

      if field in tqMatchDict:
        self.log.debug("Evaluating %s with value %s" % (field, tqMatchDict[field]))

        _, fullTableN = self.__generateTablesName(sqlTables, field)

        sqlMultiCondList = []
        csql = None

        # Now evaluating Tags
        if field == 'Tag':
          tag_fv = tqMatchDict.get('Tag')
          self.log.debug("Evaluating tag %s of type %s" % (tag_fv, type(tag_fv)))
          if isinstance(tag_fv, str):
            tag_fv = [tag_fv]

          # Is there something to consider?
          if any(fvx.lower().translate(None, string.punctuation) == 'any' for fvx in tag_fv):
            continue
          else:
            sqlMultiCondList.append(self.__generateTagSQLSubCond(fullTableN, tag_fv))

        # Now evaluating everything that is not tags
        else:
          fv = tqMatchDict.get(field)
          self.log.debug("Evaluating field %s of type %s" % (field, type(fv)))

          # Is there something to consider?
          if not fv \
                  or isinstance(fv, str) and fv.lower().translate(None, string.punctuation) == 'any' \
                  or isinstance(fv, list) \
                  and any(fvx.lower().translate(None, string.punctuation) == 'any' for fvx in fv):
            continue
          # if field != 'GridCE' or 'Site' in tqMatchDict:
          # Jobs for masked sites can be matched if they specified a GridCE
          # Site is removed from tqMatchDict if the Site is mask. In this case we want
          # that the GridCE matches explicitly so the COUNT can not be 0. In this case we skip this
          # condition
          sqlMultiCondList.append("( SELECT COUNT(%s.Value) FROM %s WHERE %s.TQId = tq.TQId ) = 0" % (fullTableN,
                                                                                                      fullTableN,
                                                                                                      fullTableN))
          sqlMultiCondList.append(self.__generateSQLSubCond("%%s IN ( SELECT %s.Value \
                                                            FROM %s \
                                                            WHERE %s.TQId = tq.TQId )" % (fullTableN,
                                                                                          fullTableN,
                                                                                          fullTableN),
                                                            tqMatchDict.get(field)))

        sqlCondList.append("( %s )" % " OR ".join(sqlMultiCondList))

        # In case of Site, check it's not in job banned sites
        if field in bannedJobMatchFields:
          fullTableN = '`tq_TQToBanned%ss`' % field
          csql = self.__generateSQLSubCond("%%s not in ( SELECT %s.Value \
                                                          FROM %s \
                                                          WHERE %s.TQId = tq.TQId )" % (fullTableN,
                                                                                        fullTableN,
                                                                                        fullTableN),
                                           tqMatchDict[field], boolOp='OR')
          sqlCondList.append(csql)

    # Add possibly RequiredTag conditions
    rtag_fv = tqMatchDict.get('RequiredTag', [])
    if isinstance(rtag_fv, str):
      rtag_fv = [rtag_fv]

    # Is there something to consider?
    if not rtag_fv or any(fv.lower().translate(None, string.punctuation) == 'any' for fv in rtag_fv):
      pass
    elif not set(rtag_fv).issubset(set(tag_fv)):
      return S_ERROR('Wrong conditions')
    else:
      self.log.debug("Evaluating RequiredTag %s" % rtag_fv)
      sqlCondList.append(self.__generateRequiredTagSQLSubCond('`tq_TQToTags`', rtag_fv))

    # Add possibly Resource banning conditions
    for field in multiValueMatchFields:
      bannedField = "Banned%s" % field

      # Is there something to consider?
      b_fv = tqMatchDict.get(bannedField)
      if not b_fv \
              or isinstance(b_fv, str) and b_fv.lower().translate(None, string.punctuation) == 'any' \
              or isinstance(b_fv, list) \
              and any(fvx.lower().translate(None, string.punctuation) == 'any' for fvx in b_fv):
        continue

      fullTableN = '`tq_TQTo%ss`' % field

      sqlCondList.append(self.__generateSQLSubCond("%%s not in ( SELECT %s.Value \
                                                      FROM %s \
                                                      WHERE %s.TQId = tq.TQId )" % (fullTableN,
                                                                                    fullTableN,
                                                                                    fullTableN),
                                                   b_fv,
                                                   boolOp='OR'))

    # Add extra negative conditions
    if negativeCond:
      sqlCondList.append(self.__generateNotSQL(negativeCond))

    # Generate the final query string
    tqSqlCmd = "SELECT tq.TQId, tq.OwnerDN, tq.OwnerGroup FROM `tq_TaskQueues` tq WHERE %s" % (
        " AND ".join(sqlCondList))

    # Apply priorities
    tqSqlCmd = "%s ORDER BY RAND() / tq.Priority ASC" % tqSqlCmd

    # Do we want a limit?
    if numQueuesToGet:
      tqSqlCmd = "%s LIMIT %s" % (tqSqlCmd, numQueuesToGet)
    return S_OK(tqSqlCmd)

  @staticmethod
  def __generateTagSQLSubCond(tableName, tagMatchList):
    """ Generate SQL condition where ALL the specified multiValue requirements must be
        present in the matching resource list
    """
    sql1 = "SELECT COUNT(%s.Value) FROM %s WHERE %s.TQId=tq.TQId" % (tableName, tableName, tableName)
    if not tagMatchList:
      sql2 = sql1 + " AND %s.Value=''" % tableName
    else:
      if isinstance(tagMatchList, (list, tuple)):
        sql2 = sql1 + " AND %s.Value in ( %s )" % (tableName, ','.join(["%s" % v for v in tagMatchList]))
      else:
        sql2 = sql1 + " AND %s.Value=%s" % (tableName, tagMatchList)
    sql = '( ' + sql1 + ' ) = (' + sql2 + ' )'
    return sql

  @staticmethod
  def __generateRequiredTagSQLSubCond(tableName, tagMatchList):
    """ Generate SQL condition where the TQ corresponds to the requirements
        of the resource
    """
    sql = "SELECT COUNT(%s.Value) FROM %s WHERE %s.TQId=tq.TQId" % (tableName, tableName, tableName)
    if isinstance(tagMatchList, (list, tuple)):
      sql = sql + " AND %s.Value in ( %s )" % (tableName, ','.join(["%s" % v for v in tagMatchList]))
      nTags = len(tagMatchList)
    else:
      sql = sql + " AND %s.Value=%s" % (tableName, tagMatchList)
      nTags = 1
    sql = '( %s ) = %s' % (sql, nTags)
    return sql

  def deleteJob(self, jobId, connObj=False):
    """
    Delete a job from the task queues
    Return S_OK( True/False ) / S_ERROR
    """
    if not connObj:
      retVal = self._getConnection()
      if not retVal['OK']:
        return S_ERROR("Can't delete job: %s" % retVal['Message'])
      connObj = retVal['Value']
    retVal = self._query(
        "SELECT t.TQId, t.OwnerDN, t.OwnerGroup \
FROM `tq_TaskQueues` t, `tq_Jobs` j \
WHERE j.JobId = %s AND t.TQId = j.TQId" %
        jobId, conn=connObj)
    if not retVal['OK']:
      return S_ERROR("Could not get job from task queue %s: %s" % (jobId, retVal['Message']))
    data = retVal['Value']
    if not data:
      return S_OK(False)
    tqId, tqOwnerDN, tqOwnerGroup = data[0]
    self.log.verbose("Deleting job", jobId)
    retVal = self._update("DELETE FROM `tq_Jobs` WHERE JobId = %s" % jobId, conn=connObj)
    if not retVal['OK']:
      return S_ERROR("Could not delete job from task queue %s: %s" % (jobId, retVal['Message']))
    if retVal['Value'] == 0:
      # No job deleted
      return S_OK(False)
    # Always return S_OK() because job has already been taken out from the TQ
    self.__deleteTQWithDelay.add(tqId, 300, (tqId, tqOwnerDN, tqOwnerGroup))
    return S_OK(True)

  def getTaskQueueForJob(self, jobId, connObj=False):
    """
    Return TaskQueue for a given Job
    Return S_OK( [TaskQueueID] ) / S_ERROR
    """
    if not connObj:
      retVal = self._getConnection()
      if not retVal['OK']:
        return S_ERROR("Can't get TQ for job: %s" % retVal['Message'])
      connObj = retVal['Value']

    retVal = self._query('SELECT TQId FROM `tq_Jobs` WHERE JobId = %s ' % jobId, conn=connObj)

    if not retVal['OK']:
      return retVal

    if not retVal['Value']:
      return S_ERROR('Not in TaskQueues')

    return S_OK(retVal['Value'][0][0])

  def getTaskQueueForJobs(self, jobIDs, connObj=False):
    """
    Return TaskQueues for a given list of Jobs
    """
    if not connObj:
      retVal = self._getConnection()
      if not retVal['OK']:
        self.log.error("Can't get TQs for a job list", retVal['Message'])
        return retVal
      connObj = retVal['Value']

    jobString = ','.join([str(x) for x in jobIDs])
    retVal = self._query('SELECT JobId,TQId FROM `tq_Jobs` WHERE JobId in (%s) ' % jobString, conn=connObj)

    if not retVal['OK']:
      return retVal

    if not retVal['Value']:
      return S_ERROR('Not in TaskQueues')

    resultDict = {}
    for jobID, tqID in retVal['Value']:
      resultDict[int(jobID)] = int(tqID)

    return S_OK(resultDict)

  def __getOwnerForTaskQueue(self, tqId, connObj=False):
    retVal = self._query("SELECT OwnerDN, OwnerGroup from `tq_TaskQueues` WHERE TQId=%s" % tqId, conn=connObj)
    if not retVal['OK']:
      return retVal
    data = retVal['Value']
    if not data:
      return S_OK(False)
    return S_OK(retVal['Value'][0])

  def __deleteTQIfEmpty(self, args):
    (tqId, tqOwnerDN, tqOwnerGroup) = args
    retries = 3
    while retries:
      retries -= 1
      result = self.deleteTaskQueueIfEmpty(tqId, tqOwnerDN, tqOwnerGroup)
      if result['OK']:
        return
    self.log.error("Could not delete TQ",
                   "%s: %s" % (tqId, result['Message']))

  def deleteTaskQueueIfEmpty(self, tqId, tqOwnerDN=False, tqOwnerGroup=False, connObj=False):
    """
    Try to delete a task queue if its empty
    """
    if not connObj:
      retVal = self._getConnection()
      if not retVal['OK']:
        self.log.error("Can't insert job", retVal['Message'])
        return retVal
      connObj = retVal['Value']
    if not tqOwnerDN or not tqOwnerGroup:
      retVal = self.__getOwnerForTaskQueue(tqId, connObj=connObj)
      if not retVal['OK']:
        return retVal
      data = retVal['Value']
      if not data:
        return S_OK(False)
      tqOwnerDN, tqOwnerGroup = data

    sqlCmd = "SELECT TQId FROM `tq_TaskQueues` WHERE Enabled >= 1 AND `tq_TaskQueues`.TQId = %s " % tqId
    sqlCmd += "AND `tq_TaskQueues`.TQId not in ( SELECT DISTINCT TQId from `tq_Jobs` )"
    retVal = self._query(sqlCmd, conn=connObj)
    if not retVal['OK']:
      self.log.error("Could not select task queue",
                     "%s : %s" % tqId, retVal['Message'])
      return retVal
    tqToDel = retVal['Value']

    if tqToDel:
      for mvField in multiValueDefFields:
        retVal = self._update("DELETE FROM `tq_TQTo%s` WHERE TQId = %s" % (mvField, tqId), conn=connObj)
        if not retVal['OK']:
          return retVal
      retVal = self._update("DELETE FROM `tq_TaskQueues` WHERE TQId = %s" % tqId, conn=connObj)
      if not retVal['OK']:
        return retVal
      self.recalculateTQSharesForEntity(tqOwnerDN, tqOwnerGroup, connObj=connObj)
      self.log.info("Deleted empty and enabled TQ", tqId)
      return S_OK()
    return S_OK(False)

  def deleteTaskQueue(self, tqId, tqOwnerDN=False, tqOwnerGroup=False, connObj=False):
    """
    Try to delete a task queue even if it has jobs
    """
    self.log.info("Deleting TQ", tqId)
    if not connObj:
      retVal = self._getConnection()
      if not retVal['OK']:
        return S_ERROR("Can't insert job: %s" % retVal['Message'])
      connObj = retVal['Value']
    if not tqOwnerDN or not tqOwnerGroup:
      retVal = self.__getOwnerForTaskQueue(tqId, connObj=connObj)
      if not retVal['OK']:
        return retVal
      data = retVal['Value']
      if not data:
        return S_OK(False)
      tqOwnerDN, tqOwnerGroup = data
    sqlCmd = "DELETE FROM `tq_TaskQueues` WHERE `tq_TaskQueues`.TQId = %s" % tqId
    retVal = self._update(sqlCmd, conn=connObj)
    if not retVal['OK']:
      return S_ERROR("Could not delete task queue %s: %s" % (tqId, retVal['Message']))
    delTQ = retVal['Value']
    sqlCmd = "DELETE FROM `tq_Jobs` WHERE `tq_Jobs`.TQId = %s" % tqId
    retVal = self._update(sqlCmd, conn=connObj)
    if not retVal['OK']:
      return S_ERROR("Could not delete task queue %s: %s" % (tqId, retVal['Message']))
    for field in multiValueDefFields:
      retVal = self._update("DELETE FROM `tq_TQTo%s` WHERE TQId = %s" % (field, tqId), conn=connObj)
      if not retVal['OK']:
        return retVal
    if delTQ > 0:
      self.recalculateTQSharesForEntity(tqOwnerDN, tqOwnerGroup, connObj=connObj)
      return S_OK(True)
    return S_OK(False)

  def getMatchingTaskQueues(self, tqMatchDict, negativeCond=False):
    """ Get the info of the task queues that match a resource
    """
    result = self.matchAndGetTaskQueue(tqMatchDict, numQueuesToGet=0, negativeCond=negativeCond)
    if not result['OK']:
      return result
    return self.retrieveTaskQueues([tqTuple[0] for tqTuple in result['Value']])

  def getNumTaskQueues(self):
    """
     Get the number of task queues in the system
    """
    sqlCmd = "SELECT COUNT( TQId ) FROM `tq_TaskQueues`"
    retVal = self._query(sqlCmd)
    if not retVal['OK']:
      return retVal
    return S_OK(retVal['Value'][0][0])

  def retrieveTaskQueues(self, tqIdList=None):
    """
    Get all the task queues
    """
    sqlSelectEntries = ["`tq_TaskQueues`.TQId", "`tq_TaskQueues`.Priority", "COUNT( `tq_Jobs`.TQId )"]
    sqlGroupEntries = ["`tq_TaskQueues`.TQId", "`tq_TaskQueues`.Priority"]
    for field in singleValueDefFields:
      sqlSelectEntries.append("`tq_TaskQueues`.%s" % field)
      sqlGroupEntries.append("`tq_TaskQueues`.%s" % field)
    sqlCmd = "SELECT %s FROM `tq_TaskQueues`, `tq_Jobs`" % ", ".join(sqlSelectEntries)
    sqlTQCond = ""
    if tqIdList is not None:
      if not tqIdList:
        # Empty list => Fast-track no matches
        return S_OK({})
      else:
        sqlTQCond += " AND `tq_TaskQueues`.TQId in ( %s )" % ", ".join([str(id_) for id_ in tqIdList])
    sqlCmd = "%s WHERE `tq_TaskQueues`.TQId = `tq_Jobs`.TQId %s GROUP BY %s" % (sqlCmd,
                                                                                sqlTQCond,
                                                                                ", ".join(sqlGroupEntries))
    retVal = self._query(sqlCmd)
    if not retVal['OK']:
      self.log.error("Can't retrieve task queues info", retVal['Message'])
      return retVal
    tqData = {}
    for record in retVal['Value']:
      tqId = record[0]
      tqData[tqId] = {'Priority': record[1], 'Jobs': record[2]}
      record = record[3:]
      for iP, _ in enumerate(singleValueDefFields):
        tqData[tqId][singleValueDefFields[iP]] = record[iP]

    tqNeedCleaning = False
    for field in multiValueDefFields:
      table = "`tq_TQTo%s`" % field
      sqlCmd = "SELECT %s.TQId, %s.Value FROM %s" % (table, table, table)
      retVal = self._query(sqlCmd)
      if not retVal['OK']:
        self.log.error("Can't retrieve task queues field",
                       "%s info: %s" % (field, retVal['Message']))
        return retVal
      for record in retVal['Value']:
        tqId = record[0]
        value = record[1]
        if tqId not in tqData:
          if tqIdList is None or tqId in tqIdList:
            self.log.verbose(
                "Task Queue is defined for a field, but does not exist: triggering a cleaning",
                "TQID: %s, field: %s" % (tqId, field))
            tqNeedCleaning = True
        else:
          if field not in tqData[tqId]:
            tqData[tqId][field] = []
          tqData[tqId][field].append(value)
    if tqNeedCleaning:
      self.cleanOrphanedTaskQueues()
    return S_OK(tqData)

  def __updateGlobalShares(self):
    """
    Update internal structure for shares
    """
    # Update group shares
    self.__groupShares = self.getGroupShares()
    # Apply corrections if enabled
    if self.isSharesCorrectionEnabled():
      result = self.getGroupsInTQs()
      if not result['OK']:
        self.log.error("Could not get groups in the TQs", result['Message'])
      activeGroups = result['Value']
      newShares = {}
      for group in activeGroups:
        if group in self.__groupShares:
          newShares[group] = self.__groupShares[group]
      newShares = self.__sharesCorrector.correctShares(newShares)
      for group in self.__groupShares:
        if group in newShares:
          self.__groupShares[group] = newShares[group]

  def recalculateTQSharesForAll(self):
    """
    Recalculate all priorities for TQ's
    """
    if self.isSharesCorrectionEnabled():
      self.log.info("Updating correctors state")
      self.__sharesCorrector.update()
    self.__updateGlobalShares()
    self.log.info("Recalculating shares for all TQs")
    retVal = self._getConnection()
    if not retVal['OK']:
      return S_ERROR("Can't insert job: %s" % retVal['Message'])
    result = self._query("SELECT DISTINCT( OwnerGroup ) FROM `tq_TaskQueues`")
    if not result['OK']:
      return result
    for group in [r[0] for r in result['Value']]:
      self.recalculateTQSharesForEntity("all", group)
    return S_OK()

  def recalculateTQSharesForEntity(self, userDN, userGroup, connObj=False):
    """
    Recalculate the shares for a userDN/userGroup combo
    """
    self.log.info("Recalculating shares",
                  "for %s@%s TQs" % (userDN, userGroup))
    if userGroup in self.__groupShares:
      share = self.__groupShares[userGroup]
    else:
      share = float(DEFAULT_GROUP_SHARE)
    if Properties.JOB_SHARING in Registry.getPropertiesForGroup(userGroup):
      # If group has JobSharing just set prio for that entry, userDN is irrelevant
      return self.__setPrioritiesForEntity(userDN, userGroup, share, connObj=connObj)

    selSQL = "SELECT OwnerDN, COUNT(OwnerDN) FROM `tq_TaskQueues` WHERE OwnerGroup='%s' GROUP BY OwnerDN" % (userGroup)
    result = self._query(selSQL, conn=connObj)
    if not result['OK']:
      return result
    # Get owners in this group and the amount of times they appear
    data = [(r[0], r[1]) for r in result['Value'] if r]
    numOwners = len(data)
    # If there are no owners do now
    if numOwners == 0:
      return S_OK()
    # Split the share amongst the number of owners
    share /= numOwners
    entitiesShares = dict([(row[0], share) for row in data])
    # If corrector is enabled let it work it's magic
    if self.isSharesCorrectionEnabled():
      entitiesShares = self.__sharesCorrector.correctShares(entitiesShares, group=userGroup)
    # Keep updating
    owners = dict(data)
    # IF the user is already known and has more than 1 tq, the rest of the users don't need to be modified
    # (The number of owners didn't change)
    if userDN in owners and owners[userDN] > 1:
      return self.__setPrioritiesForEntity(userDN, userGroup, entitiesShares[userDN], connObj=connObj)
    # Oops the number of owners may have changed so we recalculate the prio for all owners in the group
    for userDN in owners:
      self.__setPrioritiesForEntity(userDN, userGroup, entitiesShares[userDN], connObj=connObj)
    return S_OK()

  def __setPrioritiesForEntity(self, userDN, userGroup, share, connObj=False, consolidationFunc="AVG"):
    """
    Set the priority for a userDN/userGroup combo given a splitted share
    """
    self.log.info("Setting priorities", "to %s@%s TQs" % (userDN, userGroup))
    tqCond = ["t.OwnerGroup='%s'" % userGroup]
    allowBgTQs = gConfig.getValue("/Registry/Groups/%s/AllowBackgroundTQs" % userGroup, False)
    if Properties.JOB_SHARING not in Registry.getPropertiesForGroup(userGroup):
      res = self._escapeString(userDN)
      if not res['OK']:
        return res
      userDN = res['Value']
      tqCond.append("t.OwnerDN= %s " % userDN)
    tqCond.append("t.TQId = j.TQId")
    if consolidationFunc == 'AVG':
      selectSQL = "SELECT j.TQId, SUM( j.RealPriority )/COUNT(j.RealPriority) \
FROM `tq_TaskQueues` t, `tq_Jobs` j WHERE "
    elif consolidationFunc == 'SUM':
      selectSQL = "SELECT j.TQId, SUM( j.RealPriority ) FROM `tq_TaskQueues` t, `tq_Jobs` j WHERE "
    else:
      return S_ERROR("Unknown consolidation func %s for setting priorities" % consolidationFunc)
    selectSQL += " AND ".join(tqCond)
    selectSQL += " GROUP BY t.TQId"
    result = self._query(selectSQL, conn=connObj)
    if not result['OK']:
      return result

    tqDict = dict(result['Value'])
    if not tqDict:
      return S_OK()
    # Calculate Sum of priorities
    totalPrio = 0
    for k in tqDict:
      if tqDict[k] > 0.1 or not allowBgTQs:
        totalPrio += tqDict[k]
    # Update prio for each TQ
    for tqId in tqDict:
      if tqDict[tqId] > 0.1 or not allowBgTQs:
        prio = (share / totalPrio) * tqDict[tqId]
      else:
        prio = TQ_MIN_SHARE
      prio = max(prio, TQ_MIN_SHARE)
      tqDict[tqId] = prio

    # Generate groups of TQs that will have the same prio=sum(prios) maomenos
    result = self.retrieveTaskQueues(list(tqDict))
    if not result['OK']:
      return result
    allTQsData = result['Value']
    tqGroups = {}
    for tqid in allTQsData:
      tqData = allTQsData[tqid]
      for field in ('Jobs', 'Priority') + priorityIgnoredFields:
        if field in tqData:
          tqData.pop(field)
      tqHash = []
      for f in sorted(tqData):
        tqHash.append("%s:%s" % (f, tqData[f]))
      tqHash = "|".join(tqHash)
      if tqHash not in tqGroups:
        tqGroups[tqHash] = []
      tqGroups[tqHash].append(tqid)
    tqGroups = [tqGroups[td] for td in tqGroups]

    # Do the grouping
    for tqGroup in tqGroups:
      totalPrio = 0
      if len(tqGroup) < 2:
        continue
      for tqid in tqGroup:
        totalPrio += tqDict[tqid]
      for tqid in tqGroup:
        tqDict[tqid] = totalPrio

    # Group by priorities
    prioDict = {}
    for tqId in tqDict:
      prio = tqDict[tqId]
      if prio not in prioDict:
        prioDict[prio] = []
      prioDict[prio].append(tqId)

    # Execute updates
    for prio in prioDict:
      tqList = ", ".join([str(tqId) for tqId in prioDict[prio]])
      updateSQL = "UPDATE `tq_TaskQueues` SET Priority=%.4f WHERE TQId in ( %s )" % (prio, tqList)
      self._update(updateSQL, conn=connObj)
    return S_OK()

  @staticmethod
  def getGroupShares():
    """
    Get all the shares as a DICT
    """
    result = gConfig.getSections("/Registry/Groups")
    if result['OK']:
      groups = result['Value']
    else:
      groups = []
    shares = {}
    for group in groups:
      shares[group] = gConfig.getValue("/Registry/Groups/%s/JobShare" % group, DEFAULT_GROUP_SHARE)
    return shares
