diff --git a/IncBackups/IncBackupsControl.py b/IncBackups/IncBackupsControl.py index 1ef0c6c4c..5f7acd0ac 100644 --- a/IncBackups/IncBackupsControl.py +++ b/IncBackups/IncBackupsControl.py @@ -135,7 +135,9 @@ class IncJobs(multi.Thread): secret = open(path, 'r').read() return key, secret - def awsFunction(self, fType, backupPath=None, snapshotID=None, bType=None): + ## Last argument delete is set when the snapshot is to be deleted from this repo, when this argument is set, any preceding argument is not used + + def awsFunction(self, fType, backupPath=None, snapshotID=None, bType=None, delete=None): try: if fType == 'backup': key, secret = self.getAWSData() @@ -184,6 +186,25 @@ class IncJobs(multi.Thread): if result.find('restoring') == -1: logging.statusWriter(self.statusPath, 'Failed: %s. [5009]' % (result), 1) return 0 + elif delete: + + self.backupDestinations = self.jobid.destination + + key, secret = self.getAWSData() + + command = 'export AWS_ACCESS_KEY_ID=%s AWS_SECRET_ACCESS_KEY=%s && restic -r s3:s3.amazonaws.com/%s forget %s --password-file %s' % ( + key, secret, self.website, snapshotID, self.passwordFile) + + result = ProcessUtilities.outputExecutioner(command) + + if result.find('removed snapshot') == -1: + logging.statusWriter(self.statusPath, 'Failed: %s. [5009]' % (result), 1) + return 0 + + command = 'export AWS_ACCESS_KEY_ID=%s AWS_SECRET_ACCESS_KEY=%s && restic -r s3:s3.amazonaws.com/%s prune --password-file %s' % ( + key, secret, self.website, self.passwordFile) + + ProcessUtilities.outputExecutioner(command) else: self.backupDestinations = self.jobid.destination @@ -205,7 +226,10 @@ class IncJobs(multi.Thread): logging.statusWriter(self.statusPath, "%s [88][5009]" % (str(msg)), 1) return 0 - def localFunction(self, backupPath, type, restore=None): + ## Last argument delete is set when the snapshot is to be deleted from this repo, when this argument is set, any preceding argument is not used + + def localFunction(self, backupPath, type, restore=None, delete=None): + if restore == None: # Define our excludes file for use with restic backupExcludesFile = '/home/%s/backup-exclude.conf' % (self.website.domain) @@ -233,6 +257,21 @@ class IncJobs(multi.Thread): newSnapshot = JobSnapshots(job=self.jobid, type='%s:%s' % (type, backupPath), snapshotid=snapShotid, destination=self.backupDestinations) newSnapshot.save() + return 1 + elif delete: + + repoLocation = '/home/%s/incbackup' % (self.website) + + command = 'restic -r %s forget %s --password-file %s' % (repoLocation, self.jobid.snapshotid, self.passwordFile) + result = ProcessUtilities.outputExecutioner(command) + + if result.find('removed snapshot') == -1: + logging.statusWriter(self.statusPath, 'Failed: %s. [5009]' % (result), 1) + return 0 + + command = 'restic -r %s prune --password-file %s' % (repoLocation, self.passwordFile) + ProcessUtilities.outputExecutioner(command) + return 1 else: repoLocation = '/home/%s/incbackup' % (self.website) @@ -247,7 +286,9 @@ class IncJobs(multi.Thread): return 1 - def sftpFunction(self, backupPath, type, restore=None): + ## Last argument delete is set when the snapshot is to be deleted from this repo, when this argument is set, any preceding argument is not used + + def sftpFunction(self, backupPath, type, restore=None, delete=None): if restore == None: # Define our excludes file for use with restic backupExcludesFile = '/home/%s/backup-exclude.conf' % (self.website.domain) @@ -276,6 +317,17 @@ class IncJobs(multi.Thread): destination=self.backupDestinations) newSnapshot.save() return 1 + elif delete: + repoLocation = '/home/backup/%s' % (self.website) + command = 'export PATH=${PATH}:/usr/bin && restic -r %s:%s forget %s --password-file %s' % ( + self.jobid.destination, repoLocation, self.jobid.snapshotid, self.passwordFile) + result = ProcessUtilities.outputExecutioner(command) + if result.find('removed snapshot') == -1: + logging.statusWriter(self.statusPath, 'Failed: %s. [5009]' % (result), 1) + return 0 + + command = 'export PATH=${PATH}:/usr/bin && restic -r %s:%s prune --password-file %s' % (self.jobid.destination, repoLocation, self.passwordFile) + ProcessUtilities.outputExecutioner(command) else: if self.reconstruct == 'remote': repoLocation = '/home/backup/%s' % (self.website) @@ -437,6 +489,7 @@ class IncJobs(multi.Thread): def restorePoint(self): try: + self.statusPath = self.extraArgs['tempPath'] self.website = self.extraArgs['website'] jobid = self.extraArgs['jobid'] @@ -526,177 +579,29 @@ class IncJobs(multi.Thread): def prepareBackupMeta(self): try: - ######### Generating meta - - ## XML Generation - - metaFileXML = Element('metaFile') - - child = SubElement(metaFileXML, 'masterDomain') - child.text = self.website.domain - - child = SubElement(metaFileXML, 'phpSelection') - child.text = self.website.phpSelection - - child = SubElement(metaFileXML, 'externalApp') - child.text = self.website.externalApp - - childDomains = self.website.childdomains_set.all() - - databases = self.website.databases_set.all() - - ## Child domains XML - - childDomainsXML = Element('ChildDomains') - - for items in childDomains: - childDomainXML = Element('domain') - - child = SubElement(childDomainXML, 'domain') - child.text = items.domain - child = SubElement(childDomainXML, 'phpSelection') - child.text = items.phpSelection - child = SubElement(childDomainXML, 'path') - child.text = items.path - - childDomainsXML.append(childDomainXML) - - metaFileXML.append(childDomainsXML) - - ## Databases XML - - databasesXML = Element('Databases') - - for items in databases: - try: - dbuser = DBUsers.objects.get(user=items.dbUser) - userToTry = items.dbUser - except: - dbusers = DBUsers.objects.all().filter(user=items.dbUser) - for it in dbusers: - dbuser = it - break - - userToTry = mysqlUtilities.mysqlUtilities.fetchuser(items.dbName) - - if userToTry == 0 or userToTry == 1: - continue - - try: - dbuser = DBUsers.objects.get(user=userToTry) - except: - dbusers = DBUsers.objects.all().filter(user=userToTry) - for it in dbusers: - dbuser = it - break + ## Use the meta function from backup utils for future improvements. - databaseXML = Element('database') + if os.path.exists(ProcessUtilities.debugPath): + logging.writeToFile('Creating meta for %s. [IncBackupsControl.py]' % (self.website.domain)) - child = SubElement(databaseXML, 'dbName') - child.text = items.dbName - child = SubElement(databaseXML, 'dbUser') - child.text = userToTry - child = SubElement(databaseXML, 'password') - child.text = dbuser.password - - databasesXML.append(databaseXML) - - metaFileXML.append(databasesXML) - - ## Get Aliases - - aliasesXML = Element('Aliases') - - aliases = backupUtilities.getAliases(self.website.domain) - - for items in aliases: - child = SubElement(aliasesXML, 'alias') - child.text = items - - metaFileXML.append(aliasesXML) - - ## Finish Alias - - ## DNS Records XML - - try: - - dnsRecordsXML = Element("dnsrecords") - dnsRecords = DNS.getDNSRecords(self.website.domain) - - for items in dnsRecords: - dnsRecordXML = Element('dnsrecord') - - child = SubElement(dnsRecordXML, 'type') - child.text = items.type - child = SubElement(dnsRecordXML, 'name') - child.text = items.name - child = SubElement(dnsRecordXML, 'content') - child.text = items.content - child = SubElement(dnsRecordXML, 'priority') - child.text = str(items.prio) - - dnsRecordsXML.append(dnsRecordXML) - - metaFileXML.append(dnsRecordsXML) - - except BaseException as msg: - logging.statusWriter(self.statusPath, '%s. [158:prepMeta]' % (str(msg)), 1) - - ## Email accounts XML - - try: - emailRecordsXML = Element('emails') - eDomain = eDomains.objects.get(domain=self.website.domain) - emailAccounts = eDomain.eusers_set.all() - - for items in emailAccounts: - emailRecordXML = Element('emailAccount') - - child = SubElement(emailRecordXML, 'email') - child.text = items.email - child = SubElement(emailRecordXML, 'password') - child.text = items.password - - emailRecordsXML.append(emailRecordXML) - - metaFileXML.append(emailRecordsXML) - except BaseException as msg: - pass - #logging.statusWriter(self.statusPath, '%s. [warning:179:prepMeta]' % (str(msg)), 1) - - ## Email meta generated! - - def prettify(elem): - """Return a pretty-printed XML string for the Element. - """ - rough_string = ElementTree.tostring(elem, 'utf-8') - reparsed = minidom.parseString(rough_string) - return reparsed.toprettyxml(indent=" ") - - ## /home/example.com/backup/backup-example-06-50-03-Thu-Feb-2018/meta.xml -- metaPath - - metaPath = '/home/cyberpanel/%s' % (str(randint(1000, 9999))) - - xmlpretty = prettify(metaFileXML).encode('ascii', 'ignore') - metaFile = open(metaPath, 'w') - metaFile.write(xmlpretty.decode('utf-8')) - metaFile.close() - os.chmod(metaPath, 0o640) + from plogical.backupUtilities import backupUtilities + status, message, metaPath = backupUtilities.prepareBackupMeta(self.website.domain, None, None, None, 0) ## meta generated - logging.statusWriter(self.statusPath, 'Meta data is ready..', 1) - - metaPathNew = '/home/%s/meta.xml' % (self.website.domain) - command = 'mv %s %s' % (metaPath, metaPathNew) - ProcessUtilities.executioner(command) - - return 1 + if status == 1: + logging.statusWriter(self.statusPath, 'Meta data is ready..', 1) + metaPathNew = '/home/%s/meta.xml' % (self.website.domain) + command = 'mv %s %s' % (metaPath, metaPathNew) + ProcessUtilities.executioner(command) + return 1 + else: + logging.statusWriter(self.statusPath, "%s [544][5009]" % (message), 1) + return 0 except BaseException as msg: - logging.statusWriter(self.statusPath, "%s [207][5009]" % (str(msg)), 1) + logging.statusWriter(self.statusPath, "%s [548][5009]" % (str(msg)), 1) return 0 def backupData(self): @@ -728,6 +633,7 @@ class IncJobs(multi.Thread): databases = self.website.databases_set.all() for items in databases: + if mysqlUtilities.mysqlUtilities.createDatabaseBackup(items.dbName, '/home/cyberpanel') == 0: return 0 @@ -760,6 +666,7 @@ class IncJobs(multi.Thread): backupPath = '/home/vmail/%s' % (self.website.domain) if os.path.exists(backupPath): + if self.backupDestinations == 'local': if self.localFunction(backupPath, 'email') == 0: return 0 @@ -929,6 +836,8 @@ Subject: %s if self.initiateRepo() == 0: return 0 + + if self.prepareBackupMeta() == 0: return 0 @@ -958,3 +867,41 @@ Subject: %s 'Failed to delete meta file: %s. [IncJobs.createBackup.591]' % str(msg), 1) logging.statusWriter(self.statusPath, 'Completed', 1) + + ### Delete Snapshot + + def DeleteSnapShot(self, inc_job): + try: + + self.statusPath = logging.fileName + + job_snapshots = inc_job.jobsnapshots_set.all() + + ### Fetch the website name from JobSnapshot object and set these variable as they are needed in called functions below + + self.website = job_snapshots[0].job.website.domain + self.passwordFile = '/home/%s/%s' % (self.website, self.website) + + for job_snapshot in job_snapshots: + + ## Functions above use the self.jobid varilable to extract information about this snapshot, so this below variable needs to be set + + self.jobid = job_snapshot + + if self.jobid.destination == 'local': + if self.localFunction('none', 'none', 0, 1) == 0: + return 0 + elif self.jobid.destination[:4] == 'sftp': + if self.sftpFunction('none', 'none', 0, 1) == 0: + return 0 + else: + if self.awsFunction('restore', '', self.jobid.snapshotid, None, 1) == 0: + return 0 + + return 1 + + except BaseException as msg: + logging.statusWriter(self.statusPath, "%s [903:DeleteSnapShot][5009]" % (str(msg)), 1) + return 0 + + diff --git a/IncBackups/views.py b/IncBackups/views.py index 191e65b2d..bca22263d 100644 --- a/IncBackups/views.py +++ b/IncBackups/views.py @@ -405,7 +405,12 @@ def delete_backup(request): backup_id = data['backupID'] - IncJob.objects.get(id=backup_id).delete() + inc_job = IncJob.objects.get(id=backup_id) + + job = IncJobs(None, None) + job.DeleteSnapShot(inc_job) + + inc_job.delete() final_dic = {'status': 1, 'error_message': 'None'} final_json = json.dumps(final_dic) @@ -450,7 +455,6 @@ def fetch_restore_points(request): final_json = json.dumps(final_dic) return HttpResponse(final_json) - def restore_point(request): try: user_id, current_acl = _get_user_acl(request) diff --git a/plogical/backupUtilities.py b/plogical/backupUtilities.py index 3a8cc6597..bddaaca2f 100755 --- a/plogical/backupUtilities.py +++ b/plogical/backupUtilities.py @@ -67,14 +67,20 @@ class backupUtilities: self.extraArgs = extraArgs @staticmethod - def prepareBackupMeta(backupDomain, backupName, tempStoragePath, backupPath): + def prepareBackupMeta(backupDomain, backupName, tempStoragePath, backupPath, FromInner = 1): try: + connection, cursor = mysqlUtilities.mysqlUtilities.setupConnection() - status = os.path.join(backupPath, 'status') + if FromInner: + status = os.path.join(backupPath, 'status') + logging.CyberCPLogFileWriter.statusWriter(status, 'Setting up meta data..') + else: + status = '/home/cyberpanel/dummy' - logging.CyberCPLogFileWriter.statusWriter(status, 'Setting up meta data..') + if os.path.exists(ProcessUtilities.debugPath): + logging.CyberCPLogFileWriter.writeToFile('Creating meta for %s.' % (backupDomain)) website = Websites.objects.get(domain=backupDomain) @@ -278,6 +284,9 @@ class backupUtilities: metaPath = '/tmp/%s' % (str(randint(1000, 9999))) + if os.path.exists(ProcessUtilities.debugPath): + logging.CyberCPLogFileWriter.writeToFile('Path to meta file %s' % (metaPath)) + xmlpretty = prettify(metaFileXML).encode('ascii', 'ignore') metaFile = open(metaPath, 'w') metaFile.write(xmlpretty.decode()) @@ -286,19 +295,21 @@ class backupUtilities: ## meta generated - newBackup = Backups(website=website, fileName=backupName, date=time.strftime("%m.%d.%Y_%H-%M-%S"), - size=0, status=1) - newBackup.save() + if FromInner: + newBackup = Backups(website=website, fileName=backupName, date=time.strftime("%m.%d.%Y_%H-%M-%S"), + size=0, status=1) + newBackup.save() - logging.CyberCPLogFileWriter.statusWriter(status, 'Meta data is ready..') + logging.CyberCPLogFileWriter.statusWriter(status, 'Meta data is ready..') return 1, 'None', metaPath - except BaseException as msg: logging.CyberCPLogFileWriter.writeToFile("%s [207][5009]" % (str(msg))) - logging.CyberCPLogFileWriter.statusWriter(status, "%s [207][5009]" % (str(msg))) - return 0, str(msg) + if FromInner: + logging.CyberCPLogFileWriter.statusWriter(status, "%s [207][5009]" % (str(msg))) + return 0, str(msg), 'None' + @staticmethod def startBackup(tempStoragePath, backupName, backupPath, metaPath=None): @@ -610,13 +621,10 @@ class backupUtilities: if VERSION == '2.1' and BUILD == '1': logging.CyberCPLogFileWriter.writeToFile('Backup version 2.1.1 detected..') - databaseUsers = database.findall('databaseUsers') - for databaseUser in databaseUsers: dbUser = databaseUser.find('dbUser').text - if mysqlUtilities.mysqlUtilities.createDatabase(dbName, dbUser, 'cyberpanel') == 0: raise BaseException diff --git a/plogical/restoreMeta.py b/plogical/restoreMeta.py index 8a267d38a..686e172b0 100644 --- a/plogical/restoreMeta.py +++ b/plogical/restoreMeta.py @@ -1,6 +1,7 @@ #!/usr/local/CyberCP/bin/python import os.path import sys + sys.path.append('/usr/local/CyberCP') os.environ.setdefault("DJANGO_SETTINGS_MODULE", "CyberCP.settings") import django @@ -15,9 +16,11 @@ import argparse try: from plogical.virtualHostUtilities import virtualHostUtilities from plogical.mailUtilities import mailUtilities + from plogical.processUtilities import ProcessUtilities except: pass + class restoreMeta(): @staticmethod @@ -104,37 +107,65 @@ class restoreMeta(): website = Websites.objects.get(domain=masterDomain) for database in databases: + dbName = database.find('dbName').text - dbUser = database.find('dbUser').text - dbPassword = database.find('password').text - try: - dbExist = Databases.objects.get(dbName=dbName) - logging.statusWriter(statusPath, 'Database exists, changing Database password.. %s' % (dbName)) - mysqlUtilities.mysqlUtilities.changePassword(dbUser, dbPassword, 1) - if mysqlUtilities.mysqlUtilities.changePassword(dbUser, dbPassword, 1) == 0: - logging.statusWriter(statusPath, 'Failed changing password for database: %s' % (dbName)) - else: - logging.statusWriter(statusPath, 'Password successfully changed for database: %s.' % (dbName)) - except: - logging.statusWriter(statusPath, 'Database did not exist, creating new.. %s' % (dbName)) - if mysqlUtilities.mysqlUtilities.createDatabase(dbName, dbUser, "cyberpanel") == 0: - logging.statusWriter(statusPath, 'Failed the creation of database: %s' % (dbName)) - else: - logging.statusWriter(statusPath, 'Database: %s successfully created.' % (dbName)) + logging.writeToFile('Backup version 2.1.1 detected..') - mysqlUtilities.mysqlUtilities.changePassword(dbUser, dbPassword, 1) - if mysqlUtilities.mysqlUtilities.changePassword(dbUser, dbPassword, 1) == 0: - logging.statusWriter(statusPath, 'Failed changing password for database: %s' % (dbName)) - else: - logging.statusWriter(statusPath, 'Password successfully changed for database: %s.' % (dbName)) + first = 1 + databaseUsers = database.findall('databaseUsers') - try: - newDB = Databases(website=website, dbName=dbName, dbUser=dbUser) - newDB.save() - except: - pass + for databaseUser in databaseUsers: + + dbUser = databaseUser.find('dbUser').text + dbHost = databaseUser.find('dbHost').text + password = databaseUser.find('password').text + + if os.path.exists(ProcessUtilities.debugPath): + logging.writeToFile('Database user: %s' % (dbUser)) + logging.writeToFile('Database host: %s' % (dbHost)) + logging.writeToFile('Database password: %s' % (password)) + + if first: + + first = 0 + + try: + dbExist = Databases.objects.get(dbName=dbName) + logging.statusWriter(statusPath, 'Database exists, changing Database password.. %s' % (dbName)) + + if mysqlUtilities.mysqlUtilities.changePassword(dbUser, password, 1, dbHost) == 0: + logging.statusWriter(statusPath, 'Failed changing password for database: %s' % (dbName)) + else: + logging.statusWriter(statusPath, 'Password successfully changed for database: %s.' % (dbName)) + + except: + + logging.statusWriter(statusPath, 'Database did not exist, creating new.. %s' % (dbName)) + + if mysqlUtilities.mysqlUtilities.createDatabase(dbName, dbUser, "cyberpanel") == 0: + logging.statusWriter(statusPath, 'Failed the creation of database: %s' % (dbName)) + else: + logging.statusWriter(statusPath, 'Database: %s successfully created.' % (dbName)) + + mysqlUtilities.mysqlUtilities.changePassword(dbUser, password, 1) + + if mysqlUtilities.mysqlUtilities.changePassword(dbUser, password, 1) == 0: + logging.statusWriter(statusPath, 'Failed changing password for database: %s' % (dbName)) + else: + logging.statusWriter(statusPath, 'Password successfully changed for database: %s.' % (dbName)) + + try: + newDB = Databases(website=website, dbName=dbName, dbUser=dbUser) + newDB.save() + except: + pass + + ## This function will not create database, only database user is created as third value is 0 for createDB + + mysqlUtilities.mysqlUtilities.createDatabase(dbName, dbUser, password, 0, dbHost) + mysqlUtilities.mysqlUtilities.changePassword(dbUser, password, 1, dbHost) ## Databases restored