bug fix: incremental backups

IncBackups/IncBackupsControl.py

@@ -135,7 +135,9 @@ class IncJobs(multi.Thread):
         secret = open(path, 'r').read()
         return key, secret
 
-    def awsFunction(self, fType, backupPath=None, snapshotID=None, bType=None):
+    ## Last argument delete is set when the snapshot is to be deleted from this repo, when this argument is set, any preceding argument is not used
+
+    def awsFunction(self, fType, backupPath=None, snapshotID=None, bType=None, delete=None):
         try:
             if fType == 'backup':
                 key, secret = self.getAWSData()
@@ -184,6 +186,25 @@ class IncJobs(multi.Thread):
                     if result.find('restoring') == -1:
                         logging.statusWriter(self.statusPath, 'Failed: %s. [5009]' % (result), 1)
                         return 0
+                elif delete:
+
+                    self.backupDestinations = self.jobid.destination
+
+                    key, secret = self.getAWSData()
+
+                    command = 'export AWS_ACCESS_KEY_ID=%s AWS_SECRET_ACCESS_KEY=%s  && restic -r s3:s3.amazonaws.com/%s forget %s --password-file %s' % (
+                        key, secret, self.website, snapshotID, self.passwordFile)
+
+                    result = ProcessUtilities.outputExecutioner(command)
+
+                    if result.find('removed snapshot') == -1:
+                        logging.statusWriter(self.statusPath, 'Failed: %s. [5009]' % (result), 1)
+                        return 0
+
+                    command = 'export AWS_ACCESS_KEY_ID=%s AWS_SECRET_ACCESS_KEY=%s  && restic -r s3:s3.amazonaws.com/%s prune --password-file %s' % (
+                        key, secret, self.website, self.passwordFile)
+
+                    ProcessUtilities.outputExecutioner(command)
                 else:
                     self.backupDestinations = self.jobid.destination
 
@@ -205,7 +226,10 @@ class IncJobs(multi.Thread):
             logging.statusWriter(self.statusPath, "%s [88][5009]" % (str(msg)), 1)
             return 0
 
-    def localFunction(self, backupPath, type, restore=None):
+    ## Last argument delete is set when the snapshot is to be deleted from this repo, when this argument is set, any preceding argument is not used
+
+    def localFunction(self, backupPath, type, restore=None, delete=None):
+
         if restore == None:
             # Define our excludes file for use with restic
             backupExcludesFile = '/home/%s/backup-exclude.conf' % (self.website.domain)
@@ -233,6 +257,21 @@ class IncJobs(multi.Thread):
                 newSnapshot = JobSnapshots(job=self.jobid, type='%s:%s' % (type, backupPath), snapshotid=snapShotid,
                                            destination=self.backupDestinations)
             newSnapshot.save()
+            return 1
+        elif delete:
+
+            repoLocation = '/home/%s/incbackup' % (self.website)
+
+            command = 'restic -r %s forget %s --password-file %s' % (repoLocation, self.jobid.snapshotid, self.passwordFile)
+            result = ProcessUtilities.outputExecutioner(command)
+
+            if result.find('removed snapshot') == -1:
+                logging.statusWriter(self.statusPath, 'Failed: %s. [5009]' % (result), 1)
+                return 0
+
+            command = 'restic -r %s prune --password-file %s' % (repoLocation, self.passwordFile)
+            ProcessUtilities.outputExecutioner(command)
+
             return 1
         else:
             repoLocation = '/home/%s/incbackup' % (self.website)
@@ -247,7 +286,9 @@ class IncJobs(multi.Thread):
 
             return 1
 
-    def sftpFunction(self, backupPath, type, restore=None):
+    ## Last argument delete is set when the snapshot is to be deleted from this repo, when this argument is set, any preceding argument is not used
+
+    def sftpFunction(self, backupPath, type, restore=None, delete=None):
         if restore == None:
             # Define our excludes file for use with restic
             backupExcludesFile = '/home/%s/backup-exclude.conf' % (self.website.domain)
@@ -276,6 +317,17 @@ class IncJobs(multi.Thread):
                                            destination=self.backupDestinations)
             newSnapshot.save()
             return 1
+        elif delete:
+            repoLocation = '/home/backup/%s' % (self.website)
+            command = 'export PATH=${PATH}:/usr/bin && restic -r %s:%s forget %s --password-file %s' % (
+                self.jobid.destination, repoLocation, self.jobid.snapshotid, self.passwordFile)
+            result = ProcessUtilities.outputExecutioner(command)
+            if result.find('removed snapshot') == -1:
+                logging.statusWriter(self.statusPath, 'Failed: %s. [5009]' % (result), 1)
+                return 0
+
+            command = 'export PATH=${PATH}:/usr/bin && restic -r %s:%s prune --password-file %s' % (self.jobid.destination, repoLocation, self.passwordFile)
+            ProcessUtilities.outputExecutioner(command)
         else:
             if self.reconstruct == 'remote':
                 repoLocation = '/home/backup/%s' % (self.website)
@@ -437,6 +489,7 @@ class IncJobs(multi.Thread):
 
     def restorePoint(self):
         try:
+
             self.statusPath = self.extraArgs['tempPath']
             self.website = self.extraArgs['website']
             jobid = self.extraArgs['jobid']
@@ -526,177 +579,29 @@ class IncJobs(multi.Thread):
     def prepareBackupMeta(self):
         try:
 
-            ######### Generating meta
-
-            ## XML Generation
-
-            metaFileXML = Element('metaFile')
-
-            child = SubElement(metaFileXML, 'masterDomain')
-            child.text = self.website.domain
-
-            child = SubElement(metaFileXML, 'phpSelection')
-            child.text = self.website.phpSelection
-
-            child = SubElement(metaFileXML, 'externalApp')
-            child.text = self.website.externalApp
-
-            childDomains = self.website.childdomains_set.all()
-
-            databases = self.website.databases_set.all()
-
-            ## Child domains XML
-
-            childDomainsXML = Element('ChildDomains')
-
-            for items in childDomains:
-                childDomainXML = Element('domain')
-
-                child = SubElement(childDomainXML, 'domain')
-                child.text = items.domain
-                child = SubElement(childDomainXML, 'phpSelection')
-                child.text = items.phpSelection
-                child = SubElement(childDomainXML, 'path')
-                child.text = items.path
-
-                childDomainsXML.append(childDomainXML)
-
-            metaFileXML.append(childDomainsXML)
-
-            ## Databases XML
-
-            databasesXML = Element('Databases')
-
-            for items in databases:
-                try:
-                    dbuser = DBUsers.objects.get(user=items.dbUser)
-                    userToTry = items.dbUser
-                except:
-                    dbusers = DBUsers.objects.all().filter(user=items.dbUser)
-                    for it in dbusers:
-                        dbuser = it
-                        break
-
-                    userToTry = mysqlUtilities.mysqlUtilities.fetchuser(items.dbName)
-
-                    if userToTry == 0 or userToTry == 1:
-                        continue
-
-                    try:
-                        dbuser = DBUsers.objects.get(user=userToTry)
-                    except:
-                        dbusers = DBUsers.objects.all().filter(user=userToTry)
-                        for it in dbusers:
-                            dbuser = it
-                            break
+            ## Use the meta function from backup utils for future improvements.
 
 
-                databaseXML = Element('database')
+            if os.path.exists(ProcessUtilities.debugPath):
+                logging.writeToFile('Creating meta for %s. [IncBackupsControl.py]' % (self.website.domain))
 
-                child = SubElement(databaseXML, 'dbName')
-                child.text = items.dbName
-                child = SubElement(databaseXML, 'dbUser')
-                child.text = userToTry
-                child = SubElement(databaseXML, 'password')
-                child.text = dbuser.password
-
-                databasesXML.append(databaseXML)
-
-            metaFileXML.append(databasesXML)
-
-            ## Get Aliases
-
-            aliasesXML = Element('Aliases')
-
-            aliases = backupUtilities.getAliases(self.website.domain)
-
-            for items in aliases:
-                child = SubElement(aliasesXML, 'alias')
-                child.text = items
-
-            metaFileXML.append(aliasesXML)
-
-            ## Finish Alias
-
-            ## DNS Records XML
-
-            try:
-
-                dnsRecordsXML = Element("dnsrecords")
-                dnsRecords = DNS.getDNSRecords(self.website.domain)
-
-                for items in dnsRecords:
-                    dnsRecordXML = Element('dnsrecord')
-
-                    child = SubElement(dnsRecordXML, 'type')
-                    child.text = items.type
-                    child = SubElement(dnsRecordXML, 'name')
-                    child.text = items.name
-                    child = SubElement(dnsRecordXML, 'content')
-                    child.text = items.content
-                    child = SubElement(dnsRecordXML, 'priority')
-                    child.text = str(items.prio)
-
-                    dnsRecordsXML.append(dnsRecordXML)
-
-                metaFileXML.append(dnsRecordsXML)
-
-            except BaseException as msg:
-                logging.statusWriter(self.statusPath, '%s. [158:prepMeta]' % (str(msg)), 1)
-
-            ## Email accounts XML
-
-            try:
-                emailRecordsXML = Element('emails')
-                eDomain = eDomains.objects.get(domain=self.website.domain)
-                emailAccounts = eDomain.eusers_set.all()
-
-                for items in emailAccounts:
-                    emailRecordXML = Element('emailAccount')
-
-                    child = SubElement(emailRecordXML, 'email')
-                    child.text = items.email
-                    child = SubElement(emailRecordXML, 'password')
-                    child.text = items.password
-
-                    emailRecordsXML.append(emailRecordXML)
-
-                metaFileXML.append(emailRecordsXML)
-            except BaseException as msg:
-                pass
-                #logging.statusWriter(self.statusPath, '%s. [warning:179:prepMeta]' % (str(msg)), 1)
-
-            ## Email meta generated!
-
-            def prettify(elem):
-                """Return a pretty-printed XML string for the Element.
-                """
-                rough_string = ElementTree.tostring(elem, 'utf-8')
-                reparsed = minidom.parseString(rough_string)
-                return reparsed.toprettyxml(indent="  ")
-
-            ## /home/example.com/backup/backup-example-06-50-03-Thu-Feb-2018/meta.xml -- metaPath
-
-            metaPath = '/home/cyberpanel/%s' % (str(randint(1000, 9999)))
-
-            xmlpretty = prettify(metaFileXML).encode('ascii', 'ignore')
-            metaFile = open(metaPath, 'w')
-            metaFile.write(xmlpretty.decode('utf-8'))
-            metaFile.close()
-            os.chmod(metaPath, 0o640)
+            from plogical.backupUtilities import backupUtilities
+            status, message, metaPath = backupUtilities.prepareBackupMeta(self.website.domain, None, None, None, 0)
 
             ## meta generated
 
-            logging.statusWriter(self.statusPath, 'Meta data is ready..', 1)
-
-            metaPathNew = '/home/%s/meta.xml' % (self.website.domain)
-            command = 'mv %s %s' % (metaPath, metaPathNew)
-            ProcessUtilities.executioner(command)
-
-            return 1
+            if status == 1:
+                logging.statusWriter(self.statusPath, 'Meta data is ready..', 1)
+                metaPathNew = '/home/%s/meta.xml' % (self.website.domain)
+                command = 'mv %s %s' % (metaPath, metaPathNew)
+                ProcessUtilities.executioner(command)
+                return 1
+            else:
+                logging.statusWriter(self.statusPath, "%s [544][5009]" % (message), 1)
+                return 0
 
         except BaseException as msg:
-            logging.statusWriter(self.statusPath, "%s [207][5009]" % (str(msg)), 1)
+            logging.statusWriter(self.statusPath, "%s [548][5009]" % (str(msg)), 1)
             return 0
 
     def backupData(self):
@@ -728,6 +633,7 @@ class IncJobs(multi.Thread):
             databases = self.website.databases_set.all()
 
             for items in databases:
+
                 if mysqlUtilities.mysqlUtilities.createDatabaseBackup(items.dbName, '/home/cyberpanel') == 0:
                     return 0
 
@@ -760,6 +666,7 @@ class IncJobs(multi.Thread):
             backupPath = '/home/vmail/%s' % (self.website.domain)
 
             if os.path.exists(backupPath):
+
                 if self.backupDestinations == 'local':
                     if self.localFunction(backupPath, 'email') == 0:
                         return 0
@@ -929,6 +836,8 @@ Subject: %s
         if self.initiateRepo() == 0:
             return 0
 
+
+
         if self.prepareBackupMeta() == 0:
             return 0
 
@@ -958,3 +867,41 @@ Subject: %s
                                  'Failed to delete meta file: %s. [IncJobs.createBackup.591]' % str(msg), 1)
 
         logging.statusWriter(self.statusPath, 'Completed', 1)
+
+    ### Delete Snapshot
+
+    def DeleteSnapShot(self, inc_job):
+        try:
+
+            self.statusPath = logging.fileName
+
+            job_snapshots = inc_job.jobsnapshots_set.all()
+
+            ### Fetch the website name from JobSnapshot object and set these variable as they are needed in called functions below
+
+            self.website = job_snapshots[0].job.website.domain
+            self.passwordFile = '/home/%s/%s' % (self.website, self.website)
+
+            for job_snapshot in job_snapshots:
+
+                ## Functions above use the self.jobid varilable to extract information about this snapshot, so this below variable needs to be set
+
+                self.jobid = job_snapshot
+
+                if self.jobid.destination == 'local':
+                    if self.localFunction('none', 'none', 0, 1) == 0:
+                        return 0
+                elif self.jobid.destination[:4] == 'sftp':
+                    if self.sftpFunction('none', 'none', 0, 1) == 0:
+                        return 0
+                else:
+                    if self.awsFunction('restore', '', self.jobid.snapshotid, None, 1) == 0:
+                        return 0
+
+            return 1
+
+        except BaseException as msg:
+            logging.statusWriter(self.statusPath, "%s [903:DeleteSnapShot][5009]" % (str(msg)), 1)
+            return 0
+
+

IncBackups/views.py

@@ -405,7 +405,12 @@ def delete_backup(request):
 
         backup_id = data['backupID']
 
-        IncJob.objects.get(id=backup_id).delete()
+        inc_job = IncJob.objects.get(id=backup_id)
+
+        job = IncJobs(None, None)
+        job.DeleteSnapShot(inc_job)
+
+        inc_job.delete()
 
         final_dic = {'status': 1, 'error_message': 'None'}
         final_json = json.dumps(final_dic)
@@ -450,7 +455,6 @@ def fetch_restore_points(request):
         final_json = json.dumps(final_dic)
         return HttpResponse(final_json)
 
-
 def restore_point(request):
     try:
         user_id, current_acl = _get_user_acl(request)

plogical/backupUtilities.py

@@ -67,14 +67,20 @@ class backupUtilities:
         self.extraArgs = extraArgs
 
     @staticmethod
-    def prepareBackupMeta(backupDomain, backupName, tempStoragePath, backupPath):
+    def prepareBackupMeta(backupDomain, backupName, tempStoragePath, backupPath, FromInner = 1):
         try:
 
+
             connection, cursor = mysqlUtilities.mysqlUtilities.setupConnection()
 
-            status = os.path.join(backupPath, 'status')
+            if FromInner:
+                status = os.path.join(backupPath, 'status')
+                logging.CyberCPLogFileWriter.statusWriter(status, 'Setting up meta data..')
+            else:
+                status = '/home/cyberpanel/dummy'
 
-            logging.CyberCPLogFileWriter.statusWriter(status, 'Setting up meta data..')
+            if os.path.exists(ProcessUtilities.debugPath):
+                logging.CyberCPLogFileWriter.writeToFile('Creating meta for %s.' % (backupDomain))
 
             website = Websites.objects.get(domain=backupDomain)
 
@@ -278,6 +284,9 @@ class backupUtilities:
 
             metaPath = '/tmp/%s' % (str(randint(1000, 9999)))
 
+            if os.path.exists(ProcessUtilities.debugPath):
+                logging.CyberCPLogFileWriter.writeToFile('Path to meta file %s' % (metaPath))
+
             xmlpretty = prettify(metaFileXML).encode('ascii', 'ignore')
             metaFile = open(metaPath, 'w')
             metaFile.write(xmlpretty.decode())
@@ -286,19 +295,21 @@ class backupUtilities:
 
             ## meta generated
 
-            newBackup = Backups(website=website, fileName=backupName, date=time.strftime("%m.%d.%Y_%H-%M-%S"),
-                                size=0, status=1)
-            newBackup.save()
+            if FromInner:
+                newBackup = Backups(website=website, fileName=backupName, date=time.strftime("%m.%d.%Y_%H-%M-%S"),
+                                    size=0, status=1)
+                newBackup.save()
 
-            logging.CyberCPLogFileWriter.statusWriter(status, 'Meta data is ready..')
+                logging.CyberCPLogFileWriter.statusWriter(status, 'Meta data is ready..')
 
             return 1, 'None', metaPath
 
-
         except BaseException as msg:
             logging.CyberCPLogFileWriter.writeToFile("%s [207][5009]" % (str(msg)))
-            logging.CyberCPLogFileWriter.statusWriter(status, "%s [207][5009]" % (str(msg)))
-            return 0, str(msg)
+            if FromInner:
+                logging.CyberCPLogFileWriter.statusWriter(status, "%s [207][5009]" % (str(msg)))
+            return 0, str(msg), 'None'
+
 
     @staticmethod
     def startBackup(tempStoragePath, backupName, backupPath, metaPath=None):
@@ -610,13 +621,10 @@ class backupUtilities:
                 if VERSION == '2.1' and BUILD == '1':
 
                     logging.CyberCPLogFileWriter.writeToFile('Backup version 2.1.1 detected..')
-
                     databaseUsers = database.findall('databaseUsers')
-
                     for databaseUser in databaseUsers:
 
                         dbUser = databaseUser.find('dbUser').text
-
                         if mysqlUtilities.mysqlUtilities.createDatabase(dbName, dbUser, 'cyberpanel') == 0:
                             raise BaseException
 

plogical/restoreMeta.py

@@ -1,6 +1,7 @@
 #!/usr/local/CyberCP/bin/python
 import os.path
 import sys
+
 sys.path.append('/usr/local/CyberCP')
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "CyberCP.settings")
 import django
@@ -15,9 +16,11 @@ import argparse
 try:
     from plogical.virtualHostUtilities import virtualHostUtilities
     from plogical.mailUtilities import mailUtilities
+    from plogical.processUtilities import ProcessUtilities
 except:
     pass
 
+
 class restoreMeta():
 
     @staticmethod
@@ -104,37 +107,65 @@ class restoreMeta():
             website = Websites.objects.get(domain=masterDomain)
 
             for database in databases:
+
                 dbName = database.find('dbName').text
-                dbUser = database.find('dbUser').text
-                dbPassword = database.find('password').text
 
-                try:
-                    dbExist = Databases.objects.get(dbName=dbName)
-                    logging.statusWriter(statusPath, 'Database exists, changing Database password.. %s' % (dbName))
-                    mysqlUtilities.mysqlUtilities.changePassword(dbUser, dbPassword, 1)
-                    if mysqlUtilities.mysqlUtilities.changePassword(dbUser, dbPassword, 1) == 0:
-                        logging.statusWriter(statusPath, 'Failed changing password for database: %s' % (dbName))
-                    else:
-                        logging.statusWriter(statusPath, 'Password successfully changed for database: %s.' % (dbName))
-                except:
-                    logging.statusWriter(statusPath, 'Database did not exist, creating new.. %s' % (dbName))
-                    if mysqlUtilities.mysqlUtilities.createDatabase(dbName, dbUser, "cyberpanel") == 0:
-                        logging.statusWriter(statusPath, 'Failed the creation of database: %s' % (dbName))
-                    else:
-                        logging.statusWriter(statusPath, 'Database: %s successfully created.' % (dbName))
+                logging.writeToFile('Backup version 2.1.1 detected..')
 
-                    mysqlUtilities.mysqlUtilities.changePassword(dbUser, dbPassword, 1)
-                    if mysqlUtilities.mysqlUtilities.changePassword(dbUser, dbPassword, 1) == 0:
-                        logging.statusWriter(statusPath, 'Failed changing password for database: %s' % (dbName))
-                    else:
-                        logging.statusWriter(statusPath, 'Password successfully changed for database: %s.' % (dbName))
+                first = 1
 
+                databaseUsers = database.findall('databaseUsers')
 
-                    try:
-                        newDB = Databases(website=website, dbName=dbName, dbUser=dbUser)
-                        newDB.save()
-                    except:
-                        pass
+                for databaseUser in databaseUsers:
+
+                    dbUser = databaseUser.find('dbUser').text
+                    dbHost = databaseUser.find('dbHost').text
+                    password = databaseUser.find('password').text
+
+                    if os.path.exists(ProcessUtilities.debugPath):
+                        logging.writeToFile('Database user: %s' % (dbUser))
+                        logging.writeToFile('Database host: %s' % (dbHost))
+                        logging.writeToFile('Database password: %s' % (password))
+
+                    if first:
+
+                        first = 0
+
+                        try:
+                            dbExist = Databases.objects.get(dbName=dbName)
+                            logging.statusWriter(statusPath, 'Database exists, changing Database password.. %s' % (dbName))
+
+                            if mysqlUtilities.mysqlUtilities.changePassword(dbUser, password, 1, dbHost) == 0:
+                                logging.statusWriter(statusPath, 'Failed changing password for database: %s' % (dbName))
+                            else:
+                                logging.statusWriter(statusPath, 'Password successfully changed for database: %s.' % (dbName))
+
+                        except:
+
+                            logging.statusWriter(statusPath, 'Database did not exist, creating new.. %s' % (dbName))
+
+                            if mysqlUtilities.mysqlUtilities.createDatabase(dbName, dbUser, "cyberpanel") == 0:
+                                logging.statusWriter(statusPath, 'Failed the creation of database: %s' % (dbName))
+                            else:
+                                logging.statusWriter(statusPath, 'Database: %s successfully created.' % (dbName))
+
+                            mysqlUtilities.mysqlUtilities.changePassword(dbUser, password, 1)
+
+                            if mysqlUtilities.mysqlUtilities.changePassword(dbUser, password, 1) == 0:
+                                logging.statusWriter(statusPath, 'Failed changing password for database: %s' % (dbName))
+                            else:
+                                logging.statusWriter(statusPath, 'Password successfully changed for database: %s.' % (dbName))
+
+                            try:
+                                newDB = Databases(website=website, dbName=dbName, dbUser=dbUser)
+                                newDB.save()
+                            except:
+                                pass
+
+                    ## This function will not create database, only database user is created as third value is 0 for createDB
+
+                    mysqlUtilities.mysqlUtilities.createDatabase(dbName, dbUser, password, 0, dbHost)
+                    mysqlUtilities.mysqlUtilities.changePassword(dbUser, password, 1, dbHost)
 
 
             ## Databases restored