# ------------------------------------------------------------------------------
import sys, time, os, os.path, smtplib, socket, shutil
from optparse import OptionParser
import ZODB.FileStorage
import ZODB.serialize
from DateTime import DateTime
from StringIO import StringIO
from appy.gen.mail import MailConfig, sendMail
from appy.shared.utils import executeCommand
folderName = os.path.dirname(__file__)

# ------------------------------------------------------------------------------
class BackupError(Exception): pass
ERROR_CODE = 1

# ------------------------------------------------------------------------------
class ZodbBackuper:
    '''This backuper will run every night (after 00.00). Every night excepted
       Sunday, it will perform an incremental backup. Every Sunday, the script
       will pack the ZODB, perform a full backup, and, if successful, remove all
       previous (full and incremental) backups.'''
    fullBackupExts = ('.fs', '.fsz')
    allBackupExts = ('.dat', '.deltafs', '.deltafsz') + fullBackupExts
    toRemoveExts = ('.doc', '.pdf', '.odt')
    rsyncOptions = ['rsync', '-Paz', '--no-o', '--no-g', '--no-l', '--delete',
      '--quiet', '--exclude', 'Data.fs*', '--exclude', 'Z2.*']
    # --no-o and --no-g options are for preventing rsync to perform chown and
    # chgrp commands that produce errors when done via an sshfs mount.

    def __init__(self, storageLocation, backupFolder, options):
        self.storageLocation = storageLocation
        self.backupFolder = backupFolder
        self.options = options
        # Unwrap some options directly on self.
        self.repozo = options.repozo or './repozo.py'
        self.zopectl = options.zopectl or './zopectl'
        self.logFile = file(options.logFile, 'a')
        # Keep a log of the last script execution, so we can send this info by
        # email.
        self.logMem = StringIO() 
        self.emails = options.emails.split(',')
        self.tempFolder = options.tempFolder
        self.logsBackupFolder = options.logsBackupFolder
        self.filesBackupFolder = options.filesBackupFolder
        self.zopeUser = options.zopeUser
        self.keepSeconds = int(options.keepSeconds)

    def log(self, msg, type='info'):
        for logPlace in (self.logFile, self.logMem):
            logPlace.write(msg)
            logPlace.write('\n')

    def executeCommand(self, cmd):
        '''Executes command p_cmd'''
        w = self.log
        w('Executing "%s"...' % cmd)
        out, err = executeCommand(cmd)
        if out: w(out)
        if err: w(err)
        w('Done.')

    def packZodb(self):
        '''Packs the ZODB and keeps one week history'''
        storage = ZODB.FileStorage.FileStorage(self.storageLocation)
        #storage.pack(time.time()-(7*24*60*60), ZODB.serialize.referencesf)
        storage.pack(time.time()-self.keepSeconds, ZODB.serialize.referencesf)
        for fileSuffix in ('', '.index'):
            fileName = self.storageLocation + fileSuffix
            executeCommand(['chown', self.zopeUser, fileName])

    def removeDataFsOld(self):
        '''Removes the file Data.fs.old if it exists.

           In the process of packing the ZODB, an additional file Data.fs.pack
           is created, and renamed to Data.fs once finished. It means that, when
           we pack the ZODB, 3 copies of the DB can be present at the same time:
           Data.fs, Data.fs.old and Data.fs.pack. We prefer to remove the
           Data.fs.old copy to avoid missing disk space if the DB is big.
        '''
        old = self.storageLocation + '.old'
        if os.path.exists(old):
            self.log('Removing %s...' % old)
            os.remove(old)
            self.log('Done.')

    folderCreateError = 'Could not create backup folder. Backup of log ' \
                        'files will not take place. %s'
    def backupLogs(self):
        w = self.log
        if not os.path.exists(self.logsBackupFolder):
            # Try to create the folder when to store backups of the log files
            try:
                w('Try to create backup folder for logs "%s"...' % \
                  self.logsBackupFolder)
                os.mkdir(self.logsBackupFolder)
            except IOError, ioe:
                w(folderCreateError % str(ioe))
            except OSError, oe:
                w(folderCreateError % str(oe))
        if os.path.exists(self.logsBackupFolder):
            # Ok, we can make the backup of the log files.
            # Get the folder where logs lie
            logsFolder = self.options.logsFolder
            d = os.path.dirname
            j = os.path.join
            if not logsFolder:
                logsFolder = j(d(d(self.storageLocation)), 'log')
            if not os.path.isdir(logsFolder):
                w('Cannot backup log files because folder "%s" does not ' \
                  'exist. Try using option "-g".' % logsFolder)
                return
            for logFileName in os.listdir(logsFolder):
                if logFileName.endswith('.log'):
                    backupTime = DateTime().strftime('%Y_%m_%d_%H_%M')
                    parts = os.path.splitext(logFileName)
                    copyFileName = '%s.%s%s' % (parts[0], backupTime, parts[1])
                    absCopyFileName = j(self.logsBackupFolder, copyFileName)
                    absLogFileName = j(logsFolder, logFileName)
                    w('Moving "%s" to "%s"...' % (absLogFileName,
                                                  absCopyFileName))
                    shutil.copyfile(absLogFileName, absCopyFileName)
                    os.remove(absLogFileName)
                    # I do a "copy" + a "remove" instead of a "rename" because
                    # a "rename" fails if the source and dest files are on
                    # different physical devices.

    def getDate(self, dateString):
        '''Returns a DateTime instance from p_dateString, which has the form
           YYYY-MM-DD-HH-MM-SS.'''
        return DateTime('%s/%s/%s %s:%s:%s' % tuple(dateString.split('-')))

    def removeOldBackups(self):
        '''This method removes all files (full & incremental backups) that are
           older than the last full backup.'''
        w = self.log
        # Determine date of the oldest full backup
        oldestFullBackupDate = eighties = DateTime('1980/01/01')
        for backupFile in os.listdir(self.backupFolder):
            fileDate, ext = os.path.splitext(backupFile)
            if ext in self.fullBackupExts:
                # I have found a full backup
                fileDate = self.getDate(fileDate)
                if fileDate > oldestFullBackupDate:
                    oldestFullBackupDate = fileDate
        # Remove all backup files older that oldestFullBackupDate
        if oldestFullBackupDate != eighties:
            w('Last full backup date: %s' % str(oldestFullBackupDate))
        for backupFile in os.listdir(self.backupFolder):
            fileDate, ext = os.path.splitext(backupFile)
            if (ext in self.allBackupExts) and \
               (self.getDate(fileDate) < oldestFullBackupDate):
                fullFileName = '%s/%s' % (self.backupFolder, backupFile)
                w('Removing old backup file %s...' % fullFileName)
                os.remove(fullFileName)

    def sendEmails(self):
        '''Send content of self.logMem to self.emails'''
        # Extract server, port, login and password
        login = password = None
        info = self.options.smtpServer.split(':', 3)
        if len(info) == 1: # We simply have the server
            server = info[0]
            port = 25
        elif len(info) == 2: # We have server and port
            server, port = info
            port = int(port)
        else: # We also have login and password
            server, port, login, password = info
        # Secure (TLS) or not ?
        if server.endswith('*'):
            secure = True
            server = server[:-1]
        else:
            secure = False
        # Create a MailConfig instance
        cfg = MailConfig(fromEmail=self.options.fromAddress, server=server,
                       port=port, secure=secure, login=login, password=password)
        # Send the mail
        subject = 'Appy nightlife %s > %s' % \
                  (self.storageLocation, self.backupFolder)
        sendMail(cfg, self.emails, subject, self.logMem.getvalue(),log=self.log)

    def removeTempFiles(self):
        '''LibreOffice (LO) produces temp files that a Appy/Zope process tries
           do delete once he has generated it and returned it to the browser.
           But in some cases it is not possible: Zope runs with a given user
           and LO runs with root and produces files that can't be deleted by the
           user running Zope. This method removes the temp files that could not
           have been removed by Zope.'''
        w = self.log
        w('Removing temp files in "%s"...' % self.tempFolder)
        pdfCount = docCount = odtCount = 0
        for fileName in os.listdir(self.tempFolder):
            ext = os.path.splitext(fileName)[1]
            if ext in self.toRemoveExts:
                exec '%sCount += 1' % ext[1:]
                fullFileName = os.path.join(self.tempFolder, fileName)
                #w('Removing "%s"...' % fullFileName)
                try:
                    os.remove(fullFileName)
                except OSError, oe:
                    w('Could not remove "%s" (%s).' % (fullFileName, str(oe)))
        w('%d .pdf, %d .doc and %d .odt file(s) removed.' % \
          (pdfCount, docCount, odtCount))

    def run(self):
        w = self.log
        startTime = time.time()
        mode = self.options.mode
        w('\n****** Backup launched at %s (mode: %s) ******' % \
          (str(time.asctime()), mode))
        # Shutdown the Zope instance
        w('> Shutting down Zope instance...')
        self.executeCommand([self.zopectl, 'stop'])
        # Check if we are on the "full backup day"
        dayFull = self.options.dayFullBackup
        if (dayFull == 'All') or time.asctime().startswith(dayFull):
            # If mode 'zodb', let's pack the ZODB first. Afterwards it will
            # trigger a full backup.
            if mode == 'zodb':
                # As a preamble to packing the ZODB, remove Data.fs.old if
                # present.
                self.removeDataFsOld()
                w('> Day is "%s", packing the ZODB...' % dayFull)
                self.packZodb()
                w('Pack done.')
            elif mode == 'copy':
                dest = os.path.join(self.backupFolder, 'Data.fs.new')
                w('> Day is "%s", copying %s to %s...' % \
                  (dayFull, self.storageLocation, dest))
                # Perform a copy of Data.fs to the backup folder
                shutil.copyfile(self.storageLocation, dest)
                # The copy has succeeded. Remove the previous copy and rename
                # this one.
                oldDest = os.path.join(self.backupFolder, 'Data.fs')
                w('> Copy successful. Renaming %s to %s...' % (dest, oldDest))
                if os.path.exists(oldDest):
                    os.remove(oldDest)
                    w('> (Old existing backup %s was removed).' % oldDest)
                os.rename(dest, oldDest)
                w('Done.')
            # Make a backup of the log files
            w('> Make a backup of log files...')
            self.backupLogs()
            w('Log files copied.')
        else:
            if mode == 'copy':
                w('Copy mode: nothing to copy: day is not %s.' % dayFull)
        # Do the backup with repozo
        if mode == 'zodb':
            w('> Performing backup...')
            self.executeCommand([self.options.python, self.repozo, '-BvzQ',
                           '-r', self.backupFolder, '-f', self.storageLocation])
            # R-sync the DB-controlled file system
            if self.filesBackupFolder:
                filesFolder = os.path.dirname(self.storageLocation)
                w('> Rsyncing DB-controlled file system @%s...' % filesFolder)
                command = self.rsyncOptions + [filesFolder + '/',
                                               self.filesBackupFolder]
                self.executeCommand(command)
                w('Done.')
            # Remove previous full backups
            self.removeOldBackups()
        # If a command is specified, run Zope to execute this command
        if self.options.command:
            w('> Executing command "%s"...' % self.options.command)
            jobScript = '%s/job.py' % folderName
            self.executeCommand([self.zopectl, 'run', jobScript,
                                 self.options.command])
        self.removeTempFiles()
        # Start the instance again, in normal mode
        w('> Restarting Zope instance...')
        self.executeCommand([self.zopectl, 'start'])
        stopTime = time.time()
        w('Done in %d minute(s).' % ((stopTime-startTime)/60))
        if self.emails:
            self.sendEmails()
        self.logFile.close()
        print(self.logMem.getvalue())
        self.logMem.close()

# ------------------------------------------------------------------------------
class ZodbBackupScript:
    '''usage: python backup.py storageLocation backupFolder [options]
       storageLocation is the path to a ZODB database (file storage) (ie
                       /opt/ZopeInstance/var/Data.fs);
       backupFolder is a folder exclusively dedicated for storing backups
                      of the mentioned storage (ie /data/zodbbackups).'''

    weekDays = ('Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun')
    def checkArgs(self, options, args):
        '''Check that the scripts arguments are correct.'''
        # Do I have the correct number of args?
        if len(args) != 2:
            raise BackupError('Wrong number of arguments.')
        # Check storageLocation
        if not os.path.exists(args[0]) or not os.path.isfile(args[0]):
            raise BackupError('"%s" does not exist or is not a file.' % args[0])
        # Check backupFolder
        if not os.path.isdir(args[1]):
            raise BackupError('"%s" does not exist or is not a folder.'%args[1])
        # Check logs folder
        if options.logsFolder and not os.path.isdir(options.logsFolder):
            raise BackupError('"%s" is not a folder.' % options.logsFolder)
        # Check files folder
        filesFolder = options.filesBackupFolder
        if filesFolder and not os.path.isdir(filesFolder):
            raise BackupError('"%s" is not a folder.' % filesFolder)
        # Try to create a file in this folder to check if we have write
        # access in it.
        fileName = '%s/%s.tmp' % (args[1], str(time.time()))
        try:
            f = file(fileName, 'w')
            f.write('Hello.')
            f.close()
            os.remove(fileName)
        except OSError, oe:
            raise BackupError('I do not have the right to write in ' \
                              'folder "%s".' % args[1])
        # Check temp folder
        if not os.path.isdir(options.tempFolder):
            raise BackupError('Temp folder "%s" does not exist or is not ' \
                              'a folder.' % options.tempFolder)
        # Check day of week
        dfb = options.dayFullBackup
        if (dfb != 'All') and (dfb not in self.weekDays):
            raise BackupError(
                "Day of week must be one of %s or 'All'." % str(self.weekDays))
        # Check command format
        if options.command:
            parts = options.command.split(':')
            if len(parts) not in (4,5):
                raise BackupError('Command format must be ' \
                    '<ZopeAdmin><PloneInstancePath>:<ApplicationName>:' \
                    '<ToolMethodName>[:<args>]')

    def run(self):
        optParser = OptionParser(usage=ZodbBackupScript.__doc__)
        optParser.add_option("-p", "--python", dest="python",
            help="The path to the Python interpreter running Zope",
            default='python2.4',metavar="PYTHON",type='string')
        optParser.add_option("-r", "--repozo", dest="repozo",
            help="The path to repozo.py", default='', metavar="REPOZO",
            type='string')
        optParser.add_option("-z", "--zopectl", dest="zopectl",
            help="The path to Zope instance's zopectl script", default='',
            metavar="ZOPECTL", type='string')
        optParser.add_option("-l", "--logfile", dest="logFile",
            help="Log file where this script will append output (defaults to " \
                 "./backup.log)", default='./backup.log', metavar="LOGFILE",
            type='string')
        optParser.add_option("-d", "--day-full-backup", dest="dayFullBackup",
            help="Day of the week where the full backup must be performed " \
                 "(defaults to 'Sun'). Must be one of %s or 'All' if you " \
                 "want a full backup everyday." % str(self.weekDays),
            default='Sun', metavar="DAYFULLBACKUP", type='string')
        optParser.add_option("-e", "--emails", dest="emails",
            help="Comma-separated list of emails that will receive the log " \
                 "of this script.", default='', metavar="EMAILS", type='string')
        optParser.add_option("-f", "--from-address", dest="fromAddress",
            help="From address for the sent mails", default='',
            metavar="FROMADDRESS", type='string')
        optParser.add_option("-s", "--smtp-server", dest="smtpServer",
            help="SMTP server and port (ie: localhost:25) for sending mails. " \
                 "You can also embed username and password if the SMTP " \
                 "server requires authentication, ie localhost:25:myLogin:" \
                 "myPassword. If you want a secure connection to the SMTP " \
                 "server (TLS), use a star, ie secure.smtp.server*:25.",
            default='localhost:25', metavar="SMTPSERVER", type='string')
        optParser.add_option("-t", "--tempFolder", dest="tempFolder",
            help="Folder used by LibreOffice for producing temp files. " \
                 "Defaults to /tmp.", default='/tmp', metavar="TEMP",
            type='string')
        optParser.add_option("-g", "--logsFolder",dest="logsFolder",
            help="Folder where Zope log files are (typically: event.log and " \
                 "Z2.log). If no folder is provided, we will consider to " \
                 "work on a standard Zope instance and decide that the log " \
                 "folder is, from 'storageLocation', located at ../log",
            metavar="LOGSFOLDER", type='string')
        optParser.add_option("-b", "--logsBackupFolder",dest="logsBackupFolder",
            help="Folder where backups of log files (event.log and Z2.log) " \
            "will be stored.", default='./logsbackup',
            metavar="LOGSBACKUPFOLDER", type='string')
        optParser.add_option("-F", "--filesBackupFolder",
            dest="filesBackupFolder",
            help="Folder where backups of the DB-controlle file hierarchy " \
                 "will be stored.", default='',
                 metavar="FILESBACKUPFOLDER", type='string')

        optParser.add_option("-u", "--user", dest="zopeUser",
            help="User and group that must own Data.fs. Defaults to " \
                 "zope:www-data. If this script is launched by root, for " \
                 "example, when packing the ZODB this script may produce a " \
                 "new Data.fs that the user running Zope may not be able to " \
                 "read anymore. After packing, this script makes a 'chmod' " \
                 "on Data.fs.", default='zope:www-data', metavar="USER",
            type='string')
        optParser.add_option("-k", "--keep-seconds", dest="keepSeconds",
            help="Number of seconds to leave in the ZODB history when the " \
                 "ZODB is packed.", default='86400', metavar="KEEPSECONDS",
            type='string')
        optParser.add_option("-m", "--mode", dest="mode", help="Default mode, "\
            "'zodb', uses repozo for performing backups. Mode 'copy' simply " \
            "performs a copy of the database to the specified backup folder.",
            default='zodb', metavar="MODE", type='string')
        optParser.add_option("-c", "--command", dest="command",
            help="Command to execute while Zope is running. It must have the " \
            "following format: <ZopeAdmin>:<PloneInstancePath>:" \
            "<ApplicationName>:<ToolMethodName>[:<args>]. <ZopeAdmin> is the " \
            "user name of the Zope administrator; <PloneInstancePath> is the " \
            "path, within Zope, to the Plone Site object (if not at the " \
            "root of the Zope hierarchy, use '/' as folder separator); " \
            "<ApplicationName> is the name of the Appy application; " \
            "<ToolMethodName> is the name of the method to call on the tool " \
            "in this Appy application; (optional) <args> are the arguments " \
            "to give to this method (only strings are supported). Several " \
            "arguments must be separated by '*'.",  default='',
            metavar="COMMAND", type='string')
        (options, args) = optParser.parse_args()
        try:
            self.checkArgs(options, args)
            backuper = ZodbBackuper(args[0], args[1], options)
            backuper.run()
        except BackupError, be:
            sys.stderr.write(str(be) + '\nRun the script without any ' \
                                       'argument for getting help.\n')
            sys.exit(ERROR_CODE)

# ------------------------------------------------------------------------------
if __name__ == '__main__':
    ZodbBackupScript().run()
# ------------------------------------------------------------------------------
