From ffcccb57a72d52830ccec483e92f276448e539d1 Mon Sep 17 00:00:00 2001 From: Usman Nasir Date: Mon, 16 Nov 2020 11:47:57 +0500 Subject: [PATCH 01/20] remove test database --- install/installCyberPanel.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install/installCyberPanel.py b/install/installCyberPanel.py index 8e43a5aec..e0953124e 100755 --- a/install/installCyberPanel.py +++ b/install/installCyberPanel.py @@ -271,9 +271,9 @@ class InstallCyberPanel: if self.remotemysql == 'OFF': if self.distro == ubuntu: - passwordCMD = "use mysql;GRANT ALL PRIVILEGES ON *.* TO 'root'@'localhost' IDENTIFIED BY '%s';UPDATE user SET plugin='' WHERE User='root';flush privileges;" % (InstallCyberPanel.mysql_Root_password) + passwordCMD = "use mysql;DROP DATABASE IF EXISTS test;DELETE FROM mysql.db WHERE Db='test' OR Db='test\\_%%';GRANT ALL PRIVILEGES ON *.* TO 'root'@'localhost' IDENTIFIED BY '%s';UPDATE user SET plugin='' WHERE User='root';flush privileges;" % (InstallCyberPanel.mysql_Root_password) else: - passwordCMD = "use mysql;GRANT ALL PRIVILEGES ON *.* TO 'root'@'localhost' IDENTIFIED BY '%s';flush privileges;" % (InstallCyberPanel.mysql_Root_password) + passwordCMD = "use mysql;DROP DATABASE IF EXISTS test;DELETE FROM mysql.db WHERE Db='test' OR Db='test\\_%%';GRANT ALL PRIVILEGES ON *.* TO 'root'@'localhost' IDENTIFIED BY '%s';flush privileges;" % (InstallCyberPanel.mysql_Root_password) command = 'mysql -u root -e "' + passwordCMD + '"' From a12492a306b4028d89746c25a6663223c578b084 Mon Sep 17 00:00:00 2001 From: Usman Nasir Date: Mon, 16 Nov 2020 19:32:22 +0500 Subject: [PATCH 02/20] reset virtualenv version --- cyberpanel.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cyberpanel.sh b/cyberpanel.sh index c4586ff9e..a024b3c49 100644 --- a/cyberpanel.sh +++ b/cyberpanel.sh @@ -476,7 +476,8 @@ EOF check_return fi ln -s /usr/bin/pip3 /usr/bin/pip - pip install virtualenv==16.7.9 + #pip install virtualenv==16.7.9 + pip install virtualenv check_return fi @@ -497,7 +498,8 @@ EOF DEBIAN_FRONTEND=noninteractive apt install -y python3-venv check_return - pip install virtualenv==16.7.9 + #pip install virtualenv==16.7.9 + pip install virtualenv check_return fi From bb299331651f3e2f57f65698c33f9981385d8c1d Mon Sep 17 00:00:00 2001 From: AmiOta <46223934+AmiOta@users.noreply.github.com> Date: Sun, 22 Nov 2020 18:41:05 -0500 Subject: [PATCH 03/20] Duplicate SSH User command createWebsite CLI version #450 Solve #450 --- cli/cyberPanel.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/cli/cyberPanel.py b/cli/cyberPanel.py index afc260955..1ac06edd8 100755 --- a/cli/cyberPanel.py +++ b/cli/cyberPanel.py @@ -42,6 +42,17 @@ class cyberPanel: from random import randint externalApp = "".join(re.findall("[a-zA-Z]+", domainName))[:5] + str(randint(1000, 9999)) phpSelection = 'PHP ' + php + + try: + counter = 0 + _externalApp=externalApp + while True: + tWeb = Websites.objects.get(externalApp=externalApp) + externalApp = '%s%s' % (_externalApp, str(counter)) + counter = counter + 1 + except BaseException as msg: + logger.writeforCLI(str(msg), "Error", stack()[0][3]) + time.sleep(2) result = virtualHostUtilities.createVirtualHost(domainName, email, phpSelection, externalApp, ssl, dkim, openBasedir, owner, package, 0) @@ -1556,4 +1567,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() From 0251d77855dd7ce7d15d875c7a8fdf51e8805014 Mon Sep 17 00:00:00 2001 From: Usman Nasir Date: Mon, 23 Nov 2020 10:24:35 +0500 Subject: [PATCH 04/20] update venv version --- cyberpanel.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cyberpanel.sh b/cyberpanel.sh index a024b3c49..c2e199176 100644 --- a/cyberpanel.sh +++ b/cyberpanel.sh @@ -476,8 +476,8 @@ EOF check_return fi ln -s /usr/bin/pip3 /usr/bin/pip - #pip install virtualenv==16.7.9 - pip install virtualenv + pip install virtualenv==16.7.9 + #pip install virtualenv check_return fi @@ -498,8 +498,8 @@ EOF DEBIAN_FRONTEND=noninteractive apt install -y python3-venv check_return - #pip install virtualenv==16.7.9 - pip install virtualenv + pip install virtualenv==16.7.9 + #pip install virtualenv check_return fi From 06658409112326d11d5823b2b10308f21b12fff2 Mon Sep 17 00:00:00 2001 From: Usman Nasir Date: Tue, 24 Nov 2020 13:46:51 +0500 Subject: [PATCH 05/20] venv --- cyberpanel.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cyberpanel.sh b/cyberpanel.sh index c2e199176..a024b3c49 100644 --- a/cyberpanel.sh +++ b/cyberpanel.sh @@ -476,8 +476,8 @@ EOF check_return fi ln -s /usr/bin/pip3 /usr/bin/pip - pip install virtualenv==16.7.9 - #pip install virtualenv + #pip install virtualenv==16.7.9 + pip install virtualenv check_return fi @@ -498,8 +498,8 @@ EOF DEBIAN_FRONTEND=noninteractive apt install -y python3-venv check_return - pip install virtualenv==16.7.9 - #pip install virtualenv + #pip install virtualenv==16.7.9 + pip install virtualenv check_return fi From 3fa104986b92168ad2201a6233d68704f0dd72ae Mon Sep 17 00:00:00 2001 From: Usman Nasir Date: Wed, 25 Nov 2020 13:12:38 +0500 Subject: [PATCH 06/20] venv --- cyberpanel.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cyberpanel.sh b/cyberpanel.sh index a024b3c49..c2e199176 100644 --- a/cyberpanel.sh +++ b/cyberpanel.sh @@ -476,8 +476,8 @@ EOF check_return fi ln -s /usr/bin/pip3 /usr/bin/pip - #pip install virtualenv==16.7.9 - pip install virtualenv + pip install virtualenv==16.7.9 + #pip install virtualenv check_return fi @@ -498,8 +498,8 @@ EOF DEBIAN_FRONTEND=noninteractive apt install -y python3-venv check_return - #pip install virtualenv==16.7.9 - pip install virtualenv + pip install virtualenv==16.7.9 + #pip install virtualenv check_return fi From 9b81679bad103b63320be13500f753d08c5f9a05 Mon Sep 17 00:00:00 2001 From: Usman Nasir Date: Thu, 26 Nov 2020 18:13:05 +0500 Subject: [PATCH 07/20] initiate cloudbackup --- cloudAPI/cloudManager.py | 56 +++++++++++++ cloudAPI/views.py | 4 + plogical/backupUtilities.py | 159 ++++++++++++++++++++++++++++++++++-- 3 files changed, 212 insertions(+), 7 deletions(-) diff --git a/cloudAPI/cloudManager.py b/cloudAPI/cloudManager.py index b36e2a2df..b6a2d8c60 100755 --- a/cloudAPI/cloudManager.py +++ b/cloudAPI/cloudManager.py @@ -1638,3 +1638,59 @@ class CloudManager: except BaseException as msg: return self.ajaxPre(0, str(msg)) + + def SubmitCloudBackup(self): + try: + + tempStatusPath = "/home/cyberpanel/" + str(randint(1000, 9999)) + + writeToFile = open(tempStatusPath, 'w') + writeToFile.write('Starting..,0') + writeToFile.close() + + execPath = "/usr/local/CyberCP/bin/python " + virtualHostUtilities.cyberPanel + "/plogical/backupUtilities.py" + execPath = execPath + " CloudBackup --backupDomain %s --data 1 --emails 1 --databases 1 --tempStoragePath %s" % (self.data['domain'], tempStatusPath) + ProcessUtilities.popenExecutioner(execPath) + + final_dic = {'status': 1, 'tempStatusPath': tempStatusPath} + final_json = json.dumps(final_dic) + return HttpResponse(final_json) + + except BaseException as msg: + return self.ajaxPre(0, str(msg)) + + def getCurrentCloudBackups(self): + try: + + backupDomain = self.data['domainName'] + backupsPath = '/home/cyberpanel/backups/%s/' % (backupDomain) + backups = os.listdir(backupsPath) + backups.reverse() + + json_data = "[" + checker = 0 + + counter = 1 + for items in backups: + + size = str(int(int(os.path.getsize('%s/%s' % (backupsPath, items)))/int(1048576))) + + dic = {'id': counter, + 'file': items, + 'size': '%s MBs' % (size), + } + counter = counter + 1 + + if checker == 0: + json_data = json_data + json.dumps(dic) + checker = 1 + else: + json_data = json_data + ',' + json.dumps(dic) + + json_data = json_data + ']' + final_json = json.dumps({'status': 1, 'fetchStatus': 1, 'error_message': "None", "data": json_data}) + return HttpResponse(final_json) + except BaseException as msg: + final_dic = {'status': 0, 'fetchStatus': 0, 'error_message': str(msg)} + final_json = json.dumps(final_dic) + return HttpResponse(final_json) diff --git a/cloudAPI/views.py b/cloudAPI/views.py index ee7807470..18f132bea 100755 --- a/cloudAPI/views.py +++ b/cloudAPI/views.py @@ -51,6 +51,10 @@ def router(request): return cm.ReadReportDNS() elif controller == 'ResetDNSConfigurations': return cm.ResetDNSConfigurations() + elif controller == 'SubmitCloudBackup': + return cm.SubmitCloudBackup() + elif controller == 'getCurrentCloudBackups': + return cm.getCurrentCloudBackups() elif controller == 'fetchWebsites': return cm.fetchWebsites() elif controller == 'fetchWebsiteDataJSON': diff --git a/plogical/backupUtilities.py b/plogical/backupUtilities.py index c4b79db1f..a14c4e0cb 100755 --- a/plogical/backupUtilities.py +++ b/plogical/backupUtilities.py @@ -54,10 +54,15 @@ BUILD = 3 class backupUtilities: Server_root = "/usr/local/lsws" - completeKeyPath = "/home/cyberpanel/.ssh" destinationsPath = "/home/cyberpanel/destinations" licenseKey = '/usr/local/lsws/conf/license.key' + NiceDefault = '10' + CPUDefault = '40' + CloudBackupConfigPath = '/home/cyberpanel/CloudBackup.json' + + def __init__(self, extraArgs): + self.extraArgs = extraArgs @staticmethod def prepareBackupMeta(backupDomain, backupName, tempStoragePath, backupPath): @@ -1246,6 +1251,134 @@ class backupUtilities: logging.CyberCPLogFileWriter.writeToFile(str(msg) + " [getAliases]") print(0) + ### Cloud Backup functions + + def CheckIfSleepNeeded(self): + import psutil + while (1): + if int(psutil.cpu_percent()) > int(self.cpu): + logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], + 'Current CPU usage exceeds %s percent. Backup process will sleep for 10 seconds..,0' % (self.cpu)) + import time + time.sleep(10) + else: + break + + def BackupData(self): + + ### Creating the dir to store backups + self.BackupDataPath = '%s/data' % (self.BackupPath) + command = 'mkdir -p %s' % (self.BackupDataPath) + ProcessUtilities.executioner(command) + self.DataPath = '/home/%s' % (self.extraArgs['domain']) + + ## Backing up data + + self.CheckIfSleepNeeded() + + command = 'nice -n %s cp -R %s %s' % (self.nice, self.DataPath, self.BackupDataPath) + ProcessUtilities.executioner(command) + + ## Store child domains if any in json format + + DataJson = {} + childs = [] + import json + + for child in self.website.childdomains_set.all(): + childs.append(child.domain) + + DataJson['ChildDomains'] = childs + + DataJsonPath = '%s/%s' % (self.BackupPath, 'data.json') + + writeToFile = open(DataJsonPath, 'w') + writeToFile.write(json.dumps(DataJson)) + writeToFile.close() + + return 1 + + def BackupEmails(self): + + ### Creating the dir to store backups + self.BackupDataPath = '%s/emails' % (self.BackupPath) + command = 'mkdir -p %s' % (self.BackupDataPath) + ProcessUtilities.executioner(command) + self.DataPath = '/home/vmail/%s' % (self.extraArgs['domain']) + + ## Backing up data + + self.CheckIfSleepNeeded() + + command = 'nice -n %s cp -R %s %s' % (self.nice, self.DataPath, self.BackupDataPath) + ProcessUtilities.executioner(command) + + ## Store child domains if any in json format + + DataJson = {} + emailsList = [] + import json + + from mailServer.models import Domains, EUsers + emailDomain = Domains.objects.get(domainOwner=self.website) + + for emails in emailDomain.eusers_set.all(): + emailsList.append({'email': emails.email, 'password': emails.password}) + + DataJson['emails'] = emailsList + DataJsonPath = '%s/%s' % (self.BackupPath, 'emails.json') + writeToFile = open(DataJsonPath, 'w') + writeToFile.write(json.dumps(DataJson)) + writeToFile.close() + + return 1 + + def CloudBackups(self): + import json + if os.path.exists(backupUtilities.CloudBackupConfigPath): + result = json.loads(open(backupUtilities.CloudBackupConfigPath, 'r').read()) + self.nice = result['nice'] + self.cpu = result['cpu'] + else: + self.nice = backupUtilities.NiceDefault + self.cpu = backupUtilities.CPUDefault + + self.BackupPath = '/home/cyberpanel/backups/%s/backup-' % (self.extraArgs['domain']) + self.extraArgs['domain'] + "-" + time.strftime("%m.%d.%Y_%H-%M-%S") + self.website = Websites.objects.get(domain=self.extraArgs['domain']) + + command = 'mkdir -p %s' % (self.BackupPath) + ProcessUtilities.executioner(command) + + logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], + 'Starting backup generation..,0') + if self.extraArgs['data']: + logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], + 'Generating backup for your data,5') + if self.BackupData() == 0: + logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], + 'Failed to generate backups for data. [404], 0') + return 0 + + logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], + 'Data backup successfully generated,30') + + if self.extraArgs['emails']: + logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], + 'Generating backup for your emails,5') + if self.BackupEmails() == 0: + logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], + 'Failed to generate backups for emails. [404], 0') + return 0 + + logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], + 'Emails backup successfully generated,30') + + logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], 'Completed [200].') + + + + ### Cloud Backup functions ends + def submitBackupCreation(tempStoragePath, backupName, backupPath, backupDomain): try: @@ -1386,7 +1519,6 @@ def submitBackupCreation(tempStoragePath, backupName, backupPath, backupDomain): logging.CyberCPLogFileWriter.writeToFile( str(msg) + " [submitBackupCreation]") - def cancelBackupCreation(backupCancellationDomain, fileName): try: @@ -1421,7 +1553,6 @@ def cancelBackupCreation(backupCancellationDomain, fileName): str(msg) + " [cancelBackupCreation]") print("0," + str(msg)) - def submitRestore(backupFile, dir): try: @@ -1435,7 +1566,6 @@ def submitRestore(backupFile, dir): str(msg) + " [cancelBackupCreation]") print("0," + str(msg)) - def submitDestinationCreation(ipAddress, password, port='22', user='root'): setupKeys = backupUtilities.setupSSHKeys(ipAddress, password, port, user) @@ -1445,7 +1575,6 @@ def submitDestinationCreation(ipAddress, password, port='22', user='root'): else: print(setupKeys[1]) - def getConnectionStatus(ipAddress): try: checkCon = backupUtilities.checkConnection(ipAddress) @@ -1460,8 +1589,8 @@ def getConnectionStatus(ipAddress): def main(): - parser = argparse.ArgumentParser(description='CyberPanel Installer') - parser.add_argument('function', help='Specific a function to call!') + parser = argparse.ArgumentParser(description='CyberPanel Backup Generator') + parser.add_argument('function', help='Specify a function to call!') parser.add_argument('--tempStoragePath', help='') parser.add_argument('--backupName', help='!') parser.add_argument('--backupPath', help='') @@ -1485,6 +1614,13 @@ def main(): parser.add_argument('--backupFile', help='') parser.add_argument('--dir', help='') + ### For Cloud Backups + + parser.add_argument('--data', help='') + parser.add_argument('--emails', help='') + parser.add_argument('--databases', help='') + + args = parser.parse_args() if args.function == "submitBackupCreation": @@ -1501,6 +1637,15 @@ def main(): backupUtilities.startBackup(args.tempStoragePath, args.backupName, args.backupPath, args.metaPath) elif args.function == "BackupRoot": backupUtilities.BackupRoot(args.tempStoragePath, args.backupName, args.backupPath, args.metaPath) + elif args.function == 'CloudBackup': + extraArgs = {} + extraArgs['domain'] = args.backupDomain + extraArgs['tempStatusPath'] = args.tempStoragePath + extraArgs['data'] = int(args.data) + extraArgs['emails'] = int(args.emails) + extraArgs['databases'] = int(args.databases) + bu = backupUtilities(extraArgs) + bu.CloudBackups() if __name__ == "__main__": From 2b652f2b538f2cb517c102b8327423a561a027a7 Mon Sep 17 00:00:00 2001 From: Usman Nasir Date: Fri, 27 Nov 2020 22:04:29 +0500 Subject: [PATCH 08/20] improve code for cloud backups --- plogical/backupUtilities.py | 194 ++++++++++++++++++++++++++---------- plogical/mysqlUtilities.py | 1 + 2 files changed, 142 insertions(+), 53 deletions(-) diff --git a/plogical/backupUtilities.py b/plogical/backupUtilities.py index a14c4e0cb..cfcfd3141 100755 --- a/plogical/backupUtilities.py +++ b/plogical/backupUtilities.py @@ -58,7 +58,7 @@ class backupUtilities: destinationsPath = "/home/cyberpanel/destinations" licenseKey = '/usr/local/lsws/conf/license.key' NiceDefault = '10' - CPUDefault = '40' + CPUDefault = '1000' CloudBackupConfigPath = '/home/cyberpanel/CloudBackup.json' def __init__(self, extraArgs): @@ -1256,7 +1256,8 @@ class backupUtilities: def CheckIfSleepNeeded(self): import psutil while (1): - if int(psutil.cpu_percent()) > int(self.cpu): + logging.CyberCPLogFileWriter.writeToFile('Current CPU percent %s.' % (int(psutil.cpu_percent(interval=None)))) + if int(psutil.cpu_percent(interval=None)) > int(self.cpu): logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], 'Current CPU usage exceeds %s percent. Backup process will sleep for 10 seconds..,0' % (self.cpu)) import time @@ -1265,73 +1266,149 @@ class backupUtilities: break def BackupData(self): + try: - ### Creating the dir to store backups - self.BackupDataPath = '%s/data' % (self.BackupPath) - command = 'mkdir -p %s' % (self.BackupDataPath) - ProcessUtilities.executioner(command) - self.DataPath = '/home/%s' % (self.extraArgs['domain']) + ### Creating the dir to store backups + self.BackupDataPath = '%s/data' % (self.BackupPath) + command = 'mkdir -p %s' % (self.BackupDataPath) + ProcessUtilities.executioner(command) + self.DataPath = '/home/%s' % (self.extraArgs['domain']) - ## Backing up data + ## Backing up data - self.CheckIfSleepNeeded() + self.CheckIfSleepNeeded() - command = 'nice -n %s cp -R %s %s' % (self.nice, self.DataPath, self.BackupDataPath) - ProcessUtilities.executioner(command) + command = 'nice -n %s cp -R %s %s' % (self.nice, self.DataPath, self.BackupDataPath) + ProcessUtilities.executioner(command) - ## Store child domains if any in json format + ## Store child domains if any in json format - DataJson = {} - childs = [] - import json + DataJson = {} + childs = [] + import json - for child in self.website.childdomains_set.all(): - childs.append(child.domain) + for child in self.website.childdomains_set.all(): + childs.append(child.domain) - DataJson['ChildDomains'] = childs + DataJson['ChildDomains'] = childs - DataJsonPath = '%s/%s' % (self.BackupPath, 'data.json') + DataJsonPath = '%s/%s' % (self.BackupPath, 'data.json') - writeToFile = open(DataJsonPath, 'w') - writeToFile.write(json.dumps(DataJson)) - writeToFile.close() + writeToFile = open(DataJsonPath, 'w') + writeToFile.write(json.dumps(DataJson)) + writeToFile.close() - return 1 + return 1, None + except BaseException as msg: + return 0, str(msg) def BackupEmails(self): + try: - ### Creating the dir to store backups - self.BackupDataPath = '%s/emails' % (self.BackupPath) - command = 'mkdir -p %s' % (self.BackupDataPath) - ProcessUtilities.executioner(command) - self.DataPath = '/home/vmail/%s' % (self.extraArgs['domain']) + from mailServer.models import Domains, EUsers + try: + emailDomain = Domains.objects.get(domainOwner=self.website) + except: + return 1, None - ## Backing up data + ### Creating the dir to store backups + self.BackupDataPath = '%s/emails' % (self.BackupPath) + command = 'mkdir -p %s' % (self.BackupDataPath) + ProcessUtilities.executioner(command) + self.DataPath = '/home/vmail/%s' % (self.extraArgs['domain']) - self.CheckIfSleepNeeded() + ## Backing up data - command = 'nice -n %s cp -R %s %s' % (self.nice, self.DataPath, self.BackupDataPath) - ProcessUtilities.executioner(command) + self.CheckIfSleepNeeded() - ## Store child domains if any in json format + command = 'nice -n %s cp -R %s %s' % (self.nice, self.DataPath, self.BackupDataPath) + ProcessUtilities.executioner(command) - DataJson = {} - emailsList = [] - import json + ## Store child domains if any in json format - from mailServer.models import Domains, EUsers - emailDomain = Domains.objects.get(domainOwner=self.website) + DataJson = {} + emailsList = [] + import json - for emails in emailDomain.eusers_set.all(): - emailsList.append({'email': emails.email, 'password': emails.password}) + for emails in emailDomain.eusers_set.all(): + emailsList.append({'email': emails.email, 'password': emails.password}) - DataJson['emails'] = emailsList - DataJsonPath = '%s/%s' % (self.BackupPath, 'emails.json') - writeToFile = open(DataJsonPath, 'w') - writeToFile.write(json.dumps(DataJson)) - writeToFile.close() + DataJson['emails'] = emailsList + DataJsonPath = '%s/%s' % (self.BackupPath, 'emails.json') + writeToFile = open(DataJsonPath, 'w') + writeToFile.write(json.dumps(DataJson)) + writeToFile.close() - return 1 + return 1, None + except BaseException as msg: + return 0, str(msg) + + def BackupDatabases(self): + try: + + ### Creating the dir to store backups + self.BackupDataPath = '%s/databases' % (self.BackupPath) + command = 'mkdir -p %s' % (self.BackupDataPath) + ProcessUtilities.executioner(command) + + ## Backing up data + + self.CheckIfSleepNeeded() + DataJson = {} + databases = [] + import json + + for items in self.website.databases_set.all(): + try: + dbuser = DBUsers.objects.get(user=items.dbUser) + userToTry = items.dbUser + except: + try: + dbusers = DBUsers.objects.all().filter(user=items.dbUser) + userToTry = items.dbUser + for it in dbusers: + dbuser = it + break + + userToTry = mysqlUtilities.mysqlUtilities.fetchuser(items.dbName) + + if userToTry == 0 or userToTry == 1: + continue + + try: + dbuser = DBUsers.objects.get(user=userToTry) + except: + try: + dbusers = DBUsers.objects.all().filter(user=userToTry) + for it in dbusers: + dbuser = it + break + + except BaseException as msg: + logging.CyberCPLogFileWriter.writeToFile( + 'While creating backup for %s, we failed to backup database %s. Error message: %s' % ( + self.website.domain, items.dbName, str(msg))) + continue + except BaseException as msg: + logging.CyberCPLogFileWriter.writeToFile( + 'While creating backup for %s, we failed to backup database %s. Error message: %s' % ( + self.website.domain, items.dbName, str(msg))) + continue + + databases.append({'databaseName': str(items.dbName), 'databaseUser': str(userToTry), 'password': str(dbuser.password)}) + self.CheckIfSleepNeeded() + mysqlUtilities.mysqlUtilities.createDatabaseBackup(items.dbName, self.BackupDataPath) + + DataJson['databases'] = databases + DataJsonPath = '%s/%s' % (self.BackupPath, 'databases.json') + + writeToFile = open(DataJsonPath, 'w') + writeToFile.write(json.dumps(DataJson)) + writeToFile.close() + + return 1, None + except BaseException as msg: + return 0, str(msg) def CloudBackups(self): import json @@ -1354,9 +1431,10 @@ class backupUtilities: if self.extraArgs['data']: logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], 'Generating backup for your data,5') - if self.BackupData() == 0: + result = self.BackupData() + if result[0] == 0: logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], - 'Failed to generate backups for data. [404], 0') + 'Failed to generate backups for data. Error: %s. [404], 0' % (result[1] )) return 0 logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], @@ -1365,21 +1443,31 @@ class backupUtilities: if self.extraArgs['emails']: logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], 'Generating backup for your emails,5') - if self.BackupEmails() == 0: + result = self.BackupEmails() + if result[0] == 0: logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], - 'Failed to generate backups for emails. [404], 0') + 'Failed to generate backups for emails. Error: %s. [404], 0' % (result[1] )) return 0 logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], 'Emails backup successfully generated,30') + if self.extraArgs['databases']: + logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], + 'Generating backup for your databases,5') + result = self.BackupDatabases() + if result[0] == 0: + logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], + 'Failed to generate backups for databases. Error: %s. [404], 0' % (result[1] )) + return 0 + + logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], + 'Databases backups successfully generated,30') + logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], 'Completed [200].') - - ### Cloud Backup functions ends - def submitBackupCreation(tempStoragePath, backupName, backupPath, backupDomain): try: ## /home/example.com/backup/backup-example.com-02.13.2018_10-24-52 -- tempStoragePath diff --git a/plogical/mysqlUtilities.py b/plogical/mysqlUtilities.py index 62196fb47..392fc587f 100755 --- a/plogical/mysqlUtilities.py +++ b/plogical/mysqlUtilities.py @@ -228,6 +228,7 @@ class mysqlUtilities: cnfContent = """[mysqldump] user=root password=%s +max_allowed_packet=1024M [mysql] user=root password=%s From 0e99dd9449b45ec891f99e0e951ebffe28e1d7aa Mon Sep 17 00:00:00 2001 From: Usman Nasir Date: Sat, 28 Nov 2020 10:43:19 +0500 Subject: [PATCH 09/20] final compression of cloudbackups --- plogical/backupUtilities.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/plogical/backupUtilities.py b/plogical/backupUtilities.py index cfcfd3141..f803aedce 100755 --- a/plogical/backupUtilities.py +++ b/plogical/backupUtilities.py @@ -1442,7 +1442,7 @@ class backupUtilities: if self.extraArgs['emails']: logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], - 'Generating backup for your emails,5') + 'Generating backup for your emails,40') result = self.BackupEmails() if result[0] == 0: logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], @@ -1450,11 +1450,11 @@ class backupUtilities: return 0 logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], - 'Emails backup successfully generated,30') + 'Emails backup successfully generated,60') if self.extraArgs['databases']: logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], - 'Generating backup for your databases,5') + 'Generating backup for your databases,60') result = self.BackupDatabases() if result[0] == 0: logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], @@ -1464,6 +1464,15 @@ class backupUtilities: logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], 'Databases backups successfully generated,30') + logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], + 'Creating final archive..,80') + + command = 'nice -n %s tar czf %s.tar.gz -C %s %s' % (self.nice, self.BackupPath, '/home/cyberpanel/backups/%s' % (self.extraArgs['domain']), self.BackupPath.split('/')[-1]) + ProcessUtilities.executioner(command) + + command = 'rm -rf %s' % (self.BackupPath) + ProcessUtilities.executioner(command) + logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], 'Completed [200].') ### Cloud Backup functions ends From 0d22fb6dea35aa90dad3b197cc15b9afa4561401 Mon Sep 17 00:00:00 2001 From: Usman Nasir Date: Sun, 29 Nov 2020 13:01:42 +0500 Subject: [PATCH 10/20] bug fix: https://github.com/usmannasir/cyberpanel/issues/489 --- websiteFunctions/website.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/websiteFunctions/website.py b/websiteFunctions/website.py index 10b47cf85..48115127b 100755 --- a/websiteFunctions/website.py +++ b/websiteFunctions/website.py @@ -4340,6 +4340,9 @@ StrictHostKeyChecking no command = 'mkdir -p /home/%s/.ssh/' % (domain) ProcessUtilities.executioner(command) + command = 'chown %s:%s /home/%s/.ssh/' % (website.externalApp, website.externalApp, domain) + ProcessUtilities.executioner(command) + tempPath = "/home/cyberpanel/" + str(randint(1000, 9999)) writeToFile = open(tempPath, "w") From 062e4b3fdd16fae4b2bf2d1a55f479408880065d Mon Sep 17 00:00:00 2001 From: Usman Nasir Date: Tue, 1 Dec 2020 12:33:02 +0500 Subject: [PATCH 11/20] bug fix: cloudlinux --- CLManager/CageFS.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CLManager/CageFS.py b/CLManager/CageFS.py index 9127e4bc5..e42de14a8 100644 --- a/CLManager/CageFS.py +++ b/CLManager/CageFS.py @@ -93,10 +93,10 @@ class CageFS: logging.CyberCPLogFileWriter.statusWriter(ServerStatusUtil.lswsInstallStatusPath, "Reinstalling important components ..\n", 1) - command = 'yum install -y lvemanager' + command = 'yum install -y lvemanager alt-python37-devel' ServerStatusUtil.executioner(command, statusFile) - command = 'yum reinstall -y lvemanager lve-utils cagefs alt-python27-cllib' + command = 'yum reinstall -y lvemanager lve-utils cagefs' ServerStatusUtil.executioner(command, statusFile) logging.CyberCPLogFileWriter.statusWriter(ServerStatusUtil.lswsInstallStatusPath, From 1f2f62b52833cb91dcc633da93a4929a58e9810c Mon Sep 17 00:00:00 2001 From: Usman Nasir Date: Tue, 1 Dec 2020 15:52:27 +0500 Subject: [PATCH 12/20] bug fix: cloudlinux --- CLManager/CageFS.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CLManager/CageFS.py b/CLManager/CageFS.py index e42de14a8..17713a6b8 100644 --- a/CLManager/CageFS.py +++ b/CLManager/CageFS.py @@ -93,7 +93,7 @@ class CageFS: logging.CyberCPLogFileWriter.statusWriter(ServerStatusUtil.lswsInstallStatusPath, "Reinstalling important components ..\n", 1) - command = 'yum install -y lvemanager alt-python37-devel' + command = 'yum install -y alt-python37-devel' ServerStatusUtil.executioner(command, statusFile) command = 'yum reinstall -y lvemanager lve-utils cagefs' From 8c162b3cb1749b9559c24e80e5eb2b981b185636 Mon Sep 17 00:00:00 2001 From: Usman Nasir Date: Thu, 3 Dec 2020 21:56:52 +0500 Subject: [PATCH 13/20] configure how cloudbackup process is executed --- cloudAPI/cloudManager.py | 38 +++++++++++++++++++++++++++++++++++++ cloudAPI/views.py | 4 ++++ plogical/backupUtilities.py | 7 +++++-- 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/cloudAPI/cloudManager.py b/cloudAPI/cloudManager.py index b6a2d8c60..736fdba6f 100755 --- a/cloudAPI/cloudManager.py +++ b/cloudAPI/cloudManager.py @@ -1694,3 +1694,41 @@ class CloudManager: final_dic = {'status': 0, 'fetchStatus': 0, 'error_message': str(msg)} final_json = json.dumps(final_dic) return HttpResponse(final_json) + + def fetchCloudBackupSettings(self): + try: + from plogical.backupUtilities import backupUtilities + if os.path.exists(backupUtilities.CloudBackupConfigPath): + result = json.loads(open(backupUtilities.CloudBackupConfigPath, 'r').read()) + self.nice = result['nice'] + self.cpu = result['cpu'] + self.time = result['time'] + else: + self.nice = backupUtilities.NiceDefault + self.cpu = backupUtilities.CPUDefault + self.time = backupUtilities.time + + data_ret = {'status': 1, 'nice': self.nice, 'cpu': self.cpu, 'time': self.time} + json_data = json.dumps(data_ret) + return HttpResponse(json_data) + + except BaseException as msg: + data_ret = {'status': 0, 'abort': 0, 'installationProgress': "0", 'errorMessage': str(msg)} + json_data = json.dumps(data_ret) + return HttpResponse(json_data) + + def saveCloudBackupSettings(self): + try: + from plogical.backupUtilities import backupUtilities + writeToFile = open(backupUtilities.CloudBackupConfigPath, 'w') + writeToFile.write(json.dumps(self.data)) + writeToFile.close() + + data_ret = {'status': 1} + json_data = json.dumps(data_ret) + return HttpResponse(json_data) + + except BaseException as msg: + data_ret = {'status': 0, 'abort': 0, 'installationProgress': "0", 'errorMessage': str(msg)} + json_data = json.dumps(data_ret) + return HttpResponse(json_data) diff --git a/cloudAPI/views.py b/cloudAPI/views.py index 18f132bea..9e21468ce 100755 --- a/cloudAPI/views.py +++ b/cloudAPI/views.py @@ -55,6 +55,10 @@ def router(request): return cm.SubmitCloudBackup() elif controller == 'getCurrentCloudBackups': return cm.getCurrentCloudBackups() + elif controller == 'fetchCloudBackupSettings': + return cm.fetchCloudBackupSettings() + elif controller == 'saveCloudBackupSettings': + return cm.saveCloudBackupSettings() elif controller == 'fetchWebsites': return cm.fetchWebsites() elif controller == 'fetchWebsiteDataJSON': diff --git a/plogical/backupUtilities.py b/plogical/backupUtilities.py index f803aedce..cd843cbca 100755 --- a/plogical/backupUtilities.py +++ b/plogical/backupUtilities.py @@ -60,6 +60,7 @@ class backupUtilities: NiceDefault = '10' CPUDefault = '1000' CloudBackupConfigPath = '/home/cyberpanel/CloudBackup.json' + time = 10 def __init__(self, extraArgs): self.extraArgs = extraArgs @@ -1259,9 +1260,9 @@ class backupUtilities: logging.CyberCPLogFileWriter.writeToFile('Current CPU percent %s.' % (int(psutil.cpu_percent(interval=None)))) if int(psutil.cpu_percent(interval=None)) > int(self.cpu): logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], - 'Current CPU usage exceeds %s percent. Backup process will sleep for 10 seconds..,0' % (self.cpu)) + 'Current CPU usage exceeds %s percent. Backup process will sleep for %s seconds..,0' % (self.cpu, str(self.time))) import time - time.sleep(10) + time.sleep(self.time) else: break @@ -1416,9 +1417,11 @@ class backupUtilities: result = json.loads(open(backupUtilities.CloudBackupConfigPath, 'r').read()) self.nice = result['nice'] self.cpu = result['cpu'] + self.time = int(result['time']) else: self.nice = backupUtilities.NiceDefault self.cpu = backupUtilities.CPUDefault + self.time = int(backupUtilities.time) self.BackupPath = '/home/cyberpanel/backups/%s/backup-' % (self.extraArgs['domain']) + self.extraArgs['domain'] + "-" + time.strftime("%m.%d.%Y_%H-%M-%S") self.website = Websites.objects.get(domain=self.extraArgs['domain']) From e1f1f525d5d69a01a983ec8e1830b183b118d8a3 Mon Sep 17 00:00:00 2001 From: Usman Nasir Date: Sun, 6 Dec 2020 14:10:09 +0500 Subject: [PATCH 14/20] complete restore of cloudbackup --- cloudAPI/cloudManager.py | 44 ++++++++++- cloudAPI/views.py | 4 + plogical/backupUtilities.py | 144 +++++++++++++++++++++++++++++++++++- plogical/mysqlUtilities.py | 1 + 4 files changed, 188 insertions(+), 5 deletions(-) diff --git a/cloudAPI/cloudManager.py b/cloudAPI/cloudManager.py index 736fdba6f..587e63291 100755 --- a/cloudAPI/cloudManager.py +++ b/cloudAPI/cloudManager.py @@ -1664,8 +1664,11 @@ class CloudManager: backupDomain = self.data['domainName'] backupsPath = '/home/cyberpanel/backups/%s/' % (backupDomain) - backups = os.listdir(backupsPath) - backups.reverse() + try: + backups = os.listdir(backupsPath) + backups.reverse() + except: + backups = [] json_data = "[" checker = 0 @@ -1732,3 +1735,40 @@ class CloudManager: data_ret = {'status': 0, 'abort': 0, 'installationProgress': "0", 'errorMessage': str(msg)} json_data = json.dumps(data_ret) return HttpResponse(json_data) + + def deleteCloudBackup(self): + try: + + backupDomain = self.data['domainName'] + backupFile = self.data['backupFile'] + backupsPathComplete = '/home/cyberpanel/backups/%s/%s' % (backupDomain, backupFile) + + command = 'rm -f %s' % (backupsPathComplete) + ProcessUtilities.executioner(command) + + final_json = json.dumps({'status': 1, 'fetchStatus': 1, 'error_message': "None"}) + return HttpResponse(final_json) + except BaseException as msg: + final_dic = {'status': 0, 'fetchStatus': 0, 'error_message': str(msg)} + final_json = json.dumps(final_dic) + return HttpResponse(final_json) + + def SubmitCloudBackupRestore(self): + try: + + tempStatusPath = "/home/cyberpanel/" + str(randint(1000, 9999)) + + writeToFile = open(tempStatusPath, 'w') + writeToFile.write('Starting..,0') + writeToFile.close() + + execPath = "/usr/local/CyberCP/bin/python " + virtualHostUtilities.cyberPanel + "/plogical/backupUtilities.py" + execPath = execPath + " SubmitCloudBackupRestore --backupDomain %s --backupFile %s --tempStoragePath %s" % (self.data['domain'], self.data['backupFile'], tempStatusPath) + ProcessUtilities.popenExecutioner(execPath) + + final_dic = {'status': 1, 'tempStatusPath': tempStatusPath} + final_json = json.dumps(final_dic) + return HttpResponse(final_json) + + except BaseException as msg: + return self.ajaxPre(0, str(msg)) diff --git a/cloudAPI/views.py b/cloudAPI/views.py index 9e21468ce..f88431d33 100755 --- a/cloudAPI/views.py +++ b/cloudAPI/views.py @@ -59,6 +59,10 @@ def router(request): return cm.fetchCloudBackupSettings() elif controller == 'saveCloudBackupSettings': return cm.saveCloudBackupSettings() + elif controller == 'deleteCloudBackup': + return cm.deleteCloudBackup() + elif controller == 'SubmitCloudBackupRestore': + return cm.SubmitCloudBackupRestore() elif controller == 'fetchWebsites': return cm.fetchWebsites() elif controller == 'fetchWebsiteDataJSON': diff --git a/plogical/backupUtilities.py b/plogical/backupUtilities.py index cd843cbca..b61344ceb 100755 --- a/plogical/backupUtilities.py +++ b/plogical/backupUtilities.py @@ -1279,7 +1279,7 @@ class backupUtilities: self.CheckIfSleepNeeded() - command = 'nice -n %s cp -R %s %s' % (self.nice, self.DataPath, self.BackupDataPath) + command = 'nice -n %s cp -Rp %s %s' % (self.nice, self.DataPath, self.BackupDataPath) ProcessUtilities.executioner(command) ## Store child domains if any in json format @@ -1289,7 +1289,7 @@ class backupUtilities: import json for child in self.website.childdomains_set.all(): - childs.append(child.domain) + childs.append({'domain': child.domain, 'path': child.path, 'php': child.phpSelection}) DataJson['ChildDomains'] = childs @@ -1322,7 +1322,7 @@ class backupUtilities: self.CheckIfSleepNeeded() - command = 'nice -n %s cp -R %s %s' % (self.nice, self.DataPath, self.BackupDataPath) + command = 'nice -n %s cp -Rp %s %s' % (self.nice, self.DataPath, self.BackupDataPath) ProcessUtilities.executioner(command) ## Store child domains if any in json format @@ -1476,6 +1476,137 @@ class backupUtilities: command = 'rm -rf %s' % (self.BackupPath) ProcessUtilities.executioner(command) + logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], 'Completed [200].') + + ## Restore functions + + def SubmitCloudBackupRestore(self): + import json + if os.path.exists(backupUtilities.CloudBackupConfigPath): + result = json.loads(open(backupUtilities.CloudBackupConfigPath, 'r').read()) + self.nice = result['nice'] + self.cpu = result['cpu'] + self.time = int(result['time']) + else: + self.nice = backupUtilities.NiceDefault + self.cpu = backupUtilities.CPUDefault + self.time = int(backupUtilities.time) + + self.BackupPath = '/home/cyberpanel/backups/%s/%s' % (self.extraArgs['domain'], self.extraArgs['backupFile']) + self.website = Websites.objects.get(domain=self.extraArgs['domain']) + + logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], + 'Extracting main archive..,0') + + + command = 'tar -xf %s -C %s' % (self.BackupPath, '/home/cyberpanel/backups/%s/' % (self.extraArgs['domain'])) + ProcessUtilities.executioner(command) + + logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], + 'Main Archive extracted,20') + + self.extractedPath = '/home/cyberpanel/backups/%s/%s' % (self.extraArgs['domain'], self.extraArgs['backupFile'].rstrip('.tar.gz')) + + self.dataPath = '%s/data' % (self.extractedPath) + logging.CyberCPLogFileWriter.writeToFile('Data path: %s' % (self.dataPath)) + self.databasesPath = '%s/databases' % (self.extractedPath) + logging.CyberCPLogFileWriter.writeToFile('Databases path: %s' % (self.databasesPath)) + self.emailsPath = '%s/emails' % (self.extractedPath) + logging.CyberCPLogFileWriter.writeToFile('Emails path: %s' % (self.emailsPath)) + + ## Data + + if os.path.exists(self.dataPath): + try: + logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], + 'Creating child domains if any..,20') + childDomains = json.loads(open('%s/data.json' % (self.extractedPath), 'r').read())['ChildDomains'] + + for child in childDomains: + try: + ch = ChildDomains.objects.get(domain=child['domain']) + except: + logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], + 'Creating %s,20' % (child['domain'])) + virtualHostUtilities.createDomain(self.website.domain, child['domain'], child['php'], child['path'], 1, 0, 0, + self.website.admin.userName, 0, "/home/cyberpanel/" + str(randint(1000, 9999))) + + except BaseException as msg: + logging.CyberCPLogFileWriter.writeToFile('%s [SubmitCloudBackupRestore:1533]' % str(msg)) + + homePath = '/home/%s' % (self.website.domain) + command = 'rm -rf %s' % (homePath) + ProcessUtilities.executioner(command) + + command = 'mv %s/%s %s' % (self.dataPath, self.website.domain, '/home') + ProcessUtilities.executioner(command) + + from filemanager.filemanager import FileManager + + fm = FileManager(None, None) + fm.fixPermissions(self.website.domain) + + ## Emails + + if os.path.exists(self.emailsPath): + try: + logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], + 'Creating emails if any..,40') + emails = json.loads(open('%s/emails.json' % (self.extractedPath), 'r').read())['emails'] + from mailServer.models import Domains, EUsers + emailDomain = Domains.objects.get(domain=self.website.domain) + + for email in emails: + try: + eu = EUsers.objects.get(emailOwner=emailDomain, email=email['email']) + except: + logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], + 'Creating %s,40' % (email['email'])) + emailAcct = EUsers(emailOwner=emailDomain, email=email['email'], password=email['password']) + emailAcct.mail = 'maildir:/home/vmail/%s/%s/Maildir' % (self.website.domain, email['email'].split('@')[0]) + emailAcct.save() + + EmailsHome = '/home/vmail/%s' % (self.website.domain) + + command = 'rm -rf %s' % (EmailsHome) + ProcessUtilities.executioner(command) + + command = 'mv %s/%s /home/vmail' % (self.emailsPath, self.website.domain) + ProcessUtilities.executioner(command) + + command = 'chown -R vmail:vmail %s' % (EmailsHome) + ProcessUtilities.executioner(command) + + + + except BaseException as msg: + logging.CyberCPLogFileWriter.writeToFile('%s [SubmitCloudBackupRestore:1533]' % str(msg)) + + ## Databases + + logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], + 'Restoring databases if any..,70') + + databases = json.loads(open('%s/databases.json' % (self.extractedPath), 'r').read())['databases'] + + for db in databases: + + logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], + 'Restoring database %s..,70' % (db['databaseName'])) + + mysqlUtilities.mysqlUtilities.submitDBDeletion(db['databaseName']) + + if mysqlUtilities.mysqlUtilities.createDatabase(db['databaseName'], db['databaseUser'], "cyberpanel") == 0: + raise BaseException("Failed to create Databases!") + + newDB = Databases(website=self.website, dbName=db['databaseName'], dbUser=db['databaseUser']) + newDB.save() + + mysqlUtilities.mysqlUtilities.restoreDatabaseBackup(db['databaseName'], self.databasesPath, db['password']) + + + + logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], 'Completed [200].') ### Cloud Backup functions ends @@ -1746,6 +1877,13 @@ def main(): extraArgs['databases'] = int(args.databases) bu = backupUtilities(extraArgs) bu.CloudBackups() + elif args.function == 'SubmitCloudBackupRestore': + extraArgs = {} + extraArgs['domain'] = args.backupDomain + extraArgs['tempStatusPath'] = args.tempStoragePath + extraArgs['backupFile'] = args.backupFile + bu = backupUtilities(extraArgs) + bu.SubmitCloudBackupRestore() if __name__ == "__main__": diff --git a/plogical/mysqlUtilities.py b/plogical/mysqlUtilities.py index 392fc587f..81c8fb9ac 100755 --- a/plogical/mysqlUtilities.py +++ b/plogical/mysqlUtilities.py @@ -289,6 +289,7 @@ password=%s cnfContent = """[mysqldump] user=root password=%s +max_allowed_packet=1024M [mysql] user=root password=%s From 43d514c26c0897c8dea19c2be686ab207366dcaf Mon Sep 17 00:00:00 2001 From: Usman Nasir Date: Tue, 8 Dec 2020 07:35:30 +0500 Subject: [PATCH 15/20] cloud backups to aws --- cloudAPI/cloudManager.py | 7 ++- plogical/IncScheduler.py | 87 ++++++++++++++++++++++++++++++++++++- plogical/backupUtilities.py | 21 ++++++--- requirments.txt | 4 +- s3Backups/models.py | 1 + s3Backups/s3Backups.py | 86 +++++++++++++++++++++++++++++++----- 6 files changed, 184 insertions(+), 22 deletions(-) diff --git a/cloudAPI/cloudManager.py b/cloudAPI/cloudManager.py index 587e63291..d9e587350 100755 --- a/cloudAPI/cloudManager.py +++ b/cloudAPI/cloudManager.py @@ -961,9 +961,12 @@ class CloudManager: def forceRunAWSBackup(self, request): try: + request.session['userID'] = self.admin.pk - s3 = S3Backups(request, self.data, 'forceRunAWSBackup') - s3.start() + + execPath = "/usr/local/CyberCP/bin/python /usr/local/CyberCP/IncBackups/IncScheduler.py Daily" + ProcessUtilities.popenExecutioner(execPath) + return self.ajaxPre(1, None) except BaseException as msg: return self.ajaxPre(0, str(msg)) diff --git a/plogical/IncScheduler.py b/plogical/IncScheduler.py index 0da947c46..47f3a6035 100644 --- a/plogical/IncScheduler.py +++ b/plogical/IncScheduler.py @@ -18,11 +18,15 @@ from googleapiclient.discovery import build from googleapiclient.http import MediaFileUpload from plogical.backupSchedule import backupSchedule import requests -from websiteFunctions.models import NormalBackupJobs, NormalBackupSites, NormalBackupDests, NormalBackupJobLogs +from websiteFunctions.models import NormalBackupJobs, NormalBackupJobLogs + try: + from s3Backups.models import BackupPlan, BackupLogs + import boto3 from plogical.virtualHostUtilities import virtualHostUtilities from plogical.mailUtilities import mailUtilities from plogical.CyberCPLogFileWriter import CyberCPLogFileWriter as logging + from plogical.processUtilities import ProcessUtilities except: pass @@ -589,6 +593,86 @@ Automatic backup failed for %s on %s. backupjob.config = json.dumps(jobConfig) backupjob.save() + @staticmethod + def fetchAWSKeys(): + path = '/home/cyberpanel/.aws' + credentials = path + '/credentials' + + data = open(credentials, 'r').readlines() + + aws_access_key_id = data[1].split(' ')[2].strip(' ').strip('\n') + aws_secret_access_key = data[2].split(' ')[2].strip(' ').strip('\n') + region = data[3].split(' ')[2].strip(' ').strip('\n') + + return aws_access_key_id, aws_secret_access_key, region + + @staticmethod + def forceRunAWSBackup(): + try: + + plan = BackupPlan.objects.get(name='hi') + bucketName = plan.bucket.strip('\n').strip(' ') + runTime = time.strftime("%d:%m:%Y") + + aws_access_key_id, aws_secret_access_key, region = IncScheduler.fetchAWSKeys() + + client = boto3.client( + 's3', + aws_access_key_id = aws_access_key_id, + aws_secret_access_key = aws_secret_access_key, + #region_name=region + ) + + ## + + + BackupLogs(owner=plan, level='INFO', timeStamp=time.strftime("%b %d %Y, %H:%M:%S"), + msg='Starting backup process..').save() + + PlanConfig = json.loads(plan.config) + + for items in plan.websitesinplan_set.all(): + + from plogical.backupUtilities import backupUtilities + tempStatusPath = "/home/cyberpanel/" + str(randint(1000, 9999)) + extraArgs = {} + extraArgs['domain'] = items.domain + extraArgs['tempStatusPath'] = tempStatusPath + extraArgs['data'] = PlanConfig['data'] + extraArgs['emails'] = PlanConfig['emails'] + extraArgs['databases'] = PlanConfig['databases'] + bu = backupUtilities(extraArgs) + result = bu.CloudBackups() + + finalResult = open(tempStatusPath, 'r').read() + + if result[0] == 1: + key = plan.name + '/' + runTime + '/' + result[1] + client.upload_file( + result[1], + bucketName, + key, + ) + + command = 'rm -f ' + result[1] + ProcessUtilities.executioner(command) + + BackupLogs(owner=plan, level='INFO', timeStamp=time.strftime("%b %d %Y, %H:%M:%S"), + msg='Backup successful for ' + items.domain + '.').save() + else: + BackupLogs(owner=plan, level='ERROR', timeStamp=time.strftime("%b %d %Y, %H:%M:%S"), + msg='Backup failed for ' + items.domain + '. Error: ' + finalResult).save() + + plan.lastRun = runTime + plan.save() + + BackupLogs(owner=plan, level='INFO', timeStamp=time.strftime("%b %d %Y, %H:%M:%S"), + msg='Backup Process Finished.').save() + except BaseException as msg: + logging.writeToFile(str(msg) + ' [S3Backups.runBackupPlan]') + plan = BackupPlan.objects.get(name='hi') + BackupLogs(owner=plan, timeStamp=time.strftime("%b %d %Y, %H:%M:%S"), level='ERROR', msg=str(msg)).save() + def main(): @@ -602,6 +686,7 @@ def main(): IncScheduler.git(args.function) IncScheduler.checkDiskUsage() IncScheduler.startNormalBackups(args.function) + IncScheduler.forceRunAWSBackup() if __name__ == "__main__": diff --git a/plogical/backupUtilities.py b/plogical/backupUtilities.py index b61344ceb..1918fe173 100755 --- a/plogical/backupUtilities.py +++ b/plogical/backupUtilities.py @@ -1438,7 +1438,7 @@ class backupUtilities: if result[0] == 0: logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], 'Failed to generate backups for data. Error: %s. [404], 0' % (result[1] )) - return 0 + return 0, self.BackupPath logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], 'Data backup successfully generated,30') @@ -1450,7 +1450,7 @@ class backupUtilities: if result[0] == 0: logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], 'Failed to generate backups for emails. Error: %s. [404], 0' % (result[1] )) - return 0 + return 0, self.BackupPath logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], 'Emails backup successfully generated,60') @@ -1462,7 +1462,7 @@ class backupUtilities: if result[0] == 0: logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], 'Failed to generate backups for databases. Error: %s. [404], 0' % (result[1] )) - return 0 + return 0, self.BackupPath logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], 'Databases backups successfully generated,30') @@ -1476,7 +1476,17 @@ class backupUtilities: command = 'rm -rf %s' % (self.BackupPath) ProcessUtilities.executioner(command) - logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], 'Completed [200].') + finalPath = '%s.tar.gz' % (self.BackupPath) + + command = 'chown cyberpanel:cyberpanel %s' % (finalPath) + ProcessUtilities.executioner(command) + + command = 'chmod 600:600 %s' % (finalPath) + ProcessUtilities.executioner(command) + + logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], 'Completed [200].' % (self.BackupPath)) + + return 1, self.BackupPath + '.tar.gz' ## Restore functions @@ -1605,7 +1615,8 @@ class backupUtilities: mysqlUtilities.mysqlUtilities.restoreDatabaseBackup(db['databaseName'], self.databasesPath, db['password']) - + command = 'rm -rf %s' % (self.extractedPath) + ProcessUtilities.executioner(command) logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], 'Completed [200].') diff --git a/requirments.txt b/requirments.txt index afd0c78e3..66ee115f6 100755 --- a/requirments.txt +++ b/requirments.txt @@ -5,8 +5,8 @@ Babel==2.8.0 backports.ssl-match-hostname==3.7.0.1 bcrypt==3.2.0 beautifulsoup4==4.9.3 -boto3==1.16.13 -botocore==1.19.13 +boto3==1.19.30 +botocore==1.19.30 cachetools==4.1.1 certifi==2020.11.8 cffi==1.14.3 diff --git a/s3Backups/models.py b/s3Backups/models.py index be1e7cdff..251d35cee 100755 --- a/s3Backups/models.py +++ b/s3Backups/models.py @@ -13,6 +13,7 @@ class BackupPlan(models.Model): retention = models.IntegerField() type = models.CharField(max_length=5, default='AWS') lastRun = models.CharField(max_length=50, default='0:0:0') + config = models.TextField() class WebsitesInPlan(models.Model): owner = models.ForeignKey(BackupPlan,on_delete=models.CASCADE) diff --git a/s3Backups/s3Backups.py b/s3Backups/s3Backups.py index 1031d42f9..d2cb0510d 100755 --- a/s3Backups/s3Backups.py +++ b/s3Backups/s3Backups.py @@ -176,8 +176,9 @@ class S3Backups(multi.Thread): aws_access_key_id = data[1].split(' ')[2].strip(' ').strip('\n') aws_secret_access_key = data[2].split(' ')[2].strip(' ').strip('\n') + region = data[3].split(' ')[2].strip(' ').strip('\n') - return aws_access_key_id, aws_secret_access_key + return aws_access_key_id, aws_secret_access_key, region def fetchBuckets(self): try: @@ -191,12 +192,13 @@ class S3Backups(multi.Thread): return proc.ajax(0, 'Only administrators can use AWS S3 Backups.') - aws_access_key_id, aws_secret_access_key = self.fetchAWSKeys() + aws_access_key_id, aws_secret_access_key, region = self.fetchAWSKeys() s3 = boto3.resource( 's3', aws_access_key_id = aws_access_key_id, - aws_secret_access_key = aws_secret_access_key + aws_secret_access_key = aws_secret_access_key, + region_name=region ) json_data = "[" @@ -232,8 +234,28 @@ class S3Backups(multi.Thread): admin = Administrator.objects.get(pk=userID) + ## What to backup + + WTB = {} + try: + WTB['data'] = int(self.data['data']) + except: + WTB['data'] = 0 + + try: + WTB['databases'] = int(self.data['databases']) + except: + WTB['databases'] = 0 + + try: + WTB['emails'] = int(self.data['emails']) + except: + WTB['emails'] = 0 + + ### + newPlan = BackupPlan(owner=admin, name=self.data['planName'].replace(' ', ''), freq=self.data['frequency'], - retention=self.data['retenion'], bucket=self.data['bucketName']) + retention=self.data['retenion'], bucket=self.data['bucketName'], config=json.dumps(WTB)) newPlan.save() for items in self.data['websitesInPlan']: @@ -263,12 +285,16 @@ class S3Backups(multi.Thread): checker = 0 for plan in admin.backupplan_set.all(): + config = json.loads(plan.config) dic = { 'name': plan.name, 'bucket': plan.bucket, 'freq': plan.freq, 'retention': plan.retention, 'lastRun': plan.lastRun, + 'data': config['data'], + 'databases': config['databases'], + 'emails': config['emails'], } if checker == 0: @@ -374,9 +400,28 @@ class S3Backups(multi.Thread): changePlan = BackupPlan.objects.get(name=self.data['planName']) + ## What to backup + + WTB = {} + try: + WTB['data'] = int(self.data['data']) + except: + WTB['data'] = 0 + + try: + WTB['databases'] = int(self.data['databases']) + except: + WTB['databases'] = 0 + + try: + WTB['emails'] = int(self.data['emails']) + except: + WTB['emails'] = 0 + changePlan.bucket = self.data['bucketName'] changePlan.freq = self.data['frequency'] changePlan.retention = self.data['retention'] + changePlan.config = json.dumps(WTB) changePlan.save() @@ -478,15 +523,17 @@ class S3Backups(multi.Thread): try: plan = BackupPlan.objects.get(name=self.data['planName']) + logging.writeToFile(plan.config) bucketName = plan.bucket.strip('\n').strip(' ') runTime = time.strftime("%d:%m:%Y") - aws_access_key_id, aws_secret_access_key = self.fetchAWSKeys() + aws_access_key_id, aws_secret_access_key, region = self.fetchAWSKeys() client = boto3.client( 's3', aws_access_key_id = aws_access_key_id, - aws_secret_access_key = aws_secret_access_key + aws_secret_access_key = aws_secret_access_key, + #region_name=region ) @@ -533,25 +580,40 @@ class S3Backups(multi.Thread): BackupLogs(owner=plan, level='INFO', timeStamp=time.strftime("%b %d %Y, %H:%M:%S"), msg='Starting backup process..').save() + PlanConfig = json.loads(plan.config) + for items in plan.websitesinplan_set.all(): - result = self.createBackup(items.domain) - if result[0]: - key = plan.name + '/' + runTime + '/' + result[1].split('/')[-1] + ".tar.gz" + + from plogical.backupUtilities import backupUtilities + tempStatusPath = "/home/cyberpanel/" + str(randint(1000, 9999)) + extraArgs = {} + extraArgs['domain'] = items.domain + extraArgs['tempStatusPath'] = tempStatusPath + extraArgs['data'] = PlanConfig['data'] + extraArgs['emails'] = PlanConfig['emails'] + extraArgs['databases'] = PlanConfig['databases'] + bu = backupUtilities(extraArgs) + result = bu.CloudBackups() + + finalResult = open(tempStatusPath, 'r').read() + + if result[0] == 1: + key = plan.name + '/' + runTime + '/' + result[1] client.upload_file( - result[1] + ".tar.gz", + result[1], bucketName, key, Config=config, ) - command = 'rm -f ' + result[1] + ".tar.gz" + command = 'rm -f ' + result[1] ProcessUtilities.executioner(command) BackupLogs(owner=plan, level='INFO', timeStamp=time.strftime("%b %d %Y, %H:%M:%S"), msg='Backup successful for ' + items.domain + '.').save() else: BackupLogs(owner=plan, level='ERROR', timeStamp=time.strftime("%b %d %Y, %H:%M:%S"), - msg='Backup failed for ' + items.domain + '. Error: ' + result[1]).save() + msg='Backup failed for ' + items.domain + '. Error: ' + finalResult).save() plan.lastRun = runTime plan.save() From 4503b4d588d3f680a78276366fb70b5e3bcafa2a Mon Sep 17 00:00:00 2001 From: Usman Nasir Date: Tue, 8 Dec 2020 07:55:10 +0500 Subject: [PATCH 16/20] fix powertools type --- cyberpanel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cyberpanel.sh b/cyberpanel.sh index c2e199176..ae0c9a997 100644 --- a/cyberpanel.sh +++ b/cyberpanel.sh @@ -472,7 +472,7 @@ EOF if [[ $CENTOS_8 == "True" ]]; then dnf install -y wget strace net-tools curl which bc telnet htop libevent-devel gcc libattr-devel xz-devel mariadb-devel curl-devel git platform-python-devel tar socat python3 check_return - dnf --enablerepo=PowerTools install gpgme-devel -y + dnf --enablerepo=powertools install gpgme-devel -y check_return fi ln -s /usr/bin/pip3 /usr/bin/pip From cbbd327fcb06fa304f9e8f461539ab24ccb4c207 Mon Sep 17 00:00:00 2001 From: Usman Nasir Date: Tue, 8 Dec 2020 08:07:32 +0500 Subject: [PATCH 17/20] botocore update --- requirments.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirments.txt b/requirments.txt index 66ee115f6..f6a90de14 100755 --- a/requirments.txt +++ b/requirments.txt @@ -5,7 +5,7 @@ Babel==2.8.0 backports.ssl-match-hostname==3.7.0.1 bcrypt==3.2.0 beautifulsoup4==4.9.3 -boto3==1.19.30 +boto3==1.16.30 botocore==1.19.30 cachetools==4.1.1 certifi==2020.11.8 From b68a570191a828347952c505c1aaedef724dcc61 Mon Sep 17 00:00:00 2001 From: Usman Nasir Date: Tue, 8 Dec 2020 15:01:17 +0500 Subject: [PATCH 18/20] fix scheduled cloud backups --- cloudAPI/cloudManager.py | 2 +- plogical/IncScheduler.py | 123 +++++++++++++++++-- plogical/backupUtilities.py | 5 +- s3Backups/s3Backups.py | 239 ------------------------------------ 4 files changed, 113 insertions(+), 256 deletions(-) diff --git a/cloudAPI/cloudManager.py b/cloudAPI/cloudManager.py index d9e587350..5e6185bcb 100755 --- a/cloudAPI/cloudManager.py +++ b/cloudAPI/cloudManager.py @@ -964,7 +964,7 @@ class CloudManager: request.session['userID'] = self.admin.pk - execPath = "/usr/local/CyberCP/bin/python /usr/local/CyberCP/IncBackups/IncScheduler.py Daily" + execPath = "/usr/local/CyberCP/bin/python /usr/local/CyberCP/plogical/IncScheduler.py forceRunAWSBackup --planName %s" % (self.data['planName']) ProcessUtilities.popenExecutioner(execPath) return self.ajaxPre(1, None) diff --git a/plogical/IncScheduler.py b/plogical/IncScheduler.py index 47f3a6035..3915eabdd 100644 --- a/plogical/IncScheduler.py +++ b/plogical/IncScheduler.py @@ -19,6 +19,7 @@ from googleapiclient.http import MediaFileUpload from plogical.backupSchedule import backupSchedule import requests from websiteFunctions.models import NormalBackupJobs, NormalBackupJobLogs +from boto3.s3.transfer import TransferConfig try: from s3Backups.models import BackupPlan, BackupLogs @@ -607,13 +608,16 @@ Automatic backup failed for %s on %s. return aws_access_key_id, aws_secret_access_key, region @staticmethod - def forceRunAWSBackup(): + def forceRunAWSBackup(planName): try: - plan = BackupPlan.objects.get(name='hi') + plan = BackupPlan.objects.get(name=planName) bucketName = plan.bucket.strip('\n').strip(' ') runTime = time.strftime("%d:%m:%Y") + config = TransferConfig(multipart_threshold=1024 * 25, max_concurrency=10, + multipart_chunksize=1024 * 25, use_threads=True) + aws_access_key_id, aws_secret_access_key, region = IncScheduler.fetchAWSKeys() client = boto3.client( @@ -638,23 +642,25 @@ Automatic backup failed for %s on %s. extraArgs = {} extraArgs['domain'] = items.domain extraArgs['tempStatusPath'] = tempStatusPath - extraArgs['data'] = PlanConfig['data'] - extraArgs['emails'] = PlanConfig['emails'] - extraArgs['databases'] = PlanConfig['databases'] + extraArgs['data'] = int(PlanConfig['data']) + extraArgs['emails'] = int(PlanConfig['emails']) + extraArgs['databases'] = int(PlanConfig['databases']) + bu = backupUtilities(extraArgs) - result = bu.CloudBackups() + result, fileName = bu.CloudBackups() finalResult = open(tempStatusPath, 'r').read() - if result[0] == 1: - key = plan.name + '/' + runTime + '/' + result[1] + if result == 1: + key = plan.name + '/' + runTime + '/' + fileName.split('/')[-1] client.upload_file( - result[1], + fileName, bucketName, key, + Config=config ) - command = 'rm -f ' + result[1] + command = 'rm -f ' + fileName ProcessUtilities.executioner(command) BackupLogs(owner=plan, level='INFO', timeStamp=time.strftime("%b %d %Y, %H:%M:%S"), @@ -668,25 +674,118 @@ Automatic backup failed for %s on %s. BackupLogs(owner=plan, level='INFO', timeStamp=time.strftime("%b %d %Y, %H:%M:%S"), msg='Backup Process Finished.').save() + + ### + + s3 = boto3.resource( + 's3', + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + region_name=region + ) + + ts = time.time() + + for bucket in s3.buckets.all(): + for file in bucket.objects.all(): + result = float(ts - file.last_modified.timestamp()) + if result > 100.0: + file.delete() + print(result) + except BaseException as msg: logging.writeToFile(str(msg) + ' [S3Backups.runBackupPlan]') - plan = BackupPlan.objects.get(name='hi') + plan = BackupPlan.objects.get(name=planName) BackupLogs(owner=plan, timeStamp=time.strftime("%b %d %Y, %H:%M:%S"), level='ERROR', msg=str(msg)).save() + @staticmethod + def runAWSBackups(): + try: + for plan in BackupPlan.objects.all(): + lastRunDay = plan.lastRun.split(':')[0] + lastRunMonth = plan.lastRun.split(':')[1] + + if plan.freq == 'Daily' and lastRunDay != time.strftime("%d"): + IncScheduler.forceRunAWSBackup(plan.name) + else: + if lastRunMonth == time.strftime("%m"): + days = int(time.strftime("%d")) - int(lastRunDay) + if days >= 6: + IncScheduler.forceRunAWSBackup(plan.name) + else: + days = 30 - int(lastRunDay) + days = days + int(time.strftime("%d")) + if days >= 6: + IncScheduler.forceRunAWSBackup(plan.name) + + # for plan in BackupPlanDO.objects.all(): + # lastRunDay = plan.lastRun.split(':')[0] + # lastRunMonth = plan.lastRun.split(':')[1] + # + # if plan.freq == 'Daily' and lastRunDay != time.strftime("%d"): + # self.data = {} + # self.data['planName'] = plan.name + # self.forceRunAWSBackupDO() + # else: + # if lastRunMonth == time.strftime("%m"): + # days = int(time.strftime("%d")) - int(lastRunDay) + # if days >= 6: + # self.data = {} + # self.data['planName'] = plan.name + # self.forceRunAWSBackupDO() + # else: + # days = 30 - int(lastRunDay) + # days = days + int(time.strftime("%d")) + # if days >= 6: + # self.data = {} + # self.data['planName'] = plan.name + # self.forceRunAWSBackupDO() + # + # for plan in BackupPlanMINIO.objects.all(): + # lastRunDay = plan.lastRun.split(':')[0] + # lastRunMonth = plan.lastRun.split(':')[1] + # + # if plan.freq == 'Daily' and lastRunDay != time.strftime("%d"): + # self.data = {} + # self.data['planName'] = plan.name + # self.forceRunAWSBackupMINIO() + # else: + # if lastRunMonth == time.strftime("%m"): + # days = int(time.strftime("%d")) - int(lastRunDay) + # if days >= 6: + # self.data = {} + # self.data['planName'] = plan.name + # self.forceRunAWSBackupMINIO() + # else: + # days = 30 - int(lastRunDay) + # days = days + int(time.strftime("%d")) + # if days >= 6: + # self.data = {} + # self.data['planName'] = plan.name + # self.forceRunAWSBackupMINIO() + + except BaseException as msg: + logging.writeToFile(str(msg) + ' [S3Backups.runAWSBackups]') + def main(): parser = argparse.ArgumentParser(description='CyberPanel Installer') parser.add_argument('function', help='Specific a function to call!') + parser.add_argument('--planName', help='Plan name for AWS!') args = parser.parse_args() + if args.function == 'forceRunAWSBackup': + IncScheduler.forceRunAWSBackup(args.planName) + return 0 + IncScheduler.startBackup(args.function) IncScheduler.runGoogleDriveBackups(args.function) IncScheduler.git(args.function) IncScheduler.checkDiskUsage() IncScheduler.startNormalBackups(args.function) - IncScheduler.forceRunAWSBackup() + IncScheduler.runAWSBackups() if __name__ == "__main__": diff --git a/plogical/backupUtilities.py b/plogical/backupUtilities.py index 1918fe173..813d770bf 100755 --- a/plogical/backupUtilities.py +++ b/plogical/backupUtilities.py @@ -1484,7 +1484,7 @@ class backupUtilities: command = 'chmod 600:600 %s' % (finalPath) ProcessUtilities.executioner(command) - logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], 'Completed [200].' % (self.BackupPath)) + logging.CyberCPLogFileWriter.statusWriter(self.extraArgs['tempStatusPath'], 'Completed [200].') return 1, self.BackupPath + '.tar.gz' @@ -1518,11 +1518,8 @@ class backupUtilities: self.extractedPath = '/home/cyberpanel/backups/%s/%s' % (self.extraArgs['domain'], self.extraArgs['backupFile'].rstrip('.tar.gz')) self.dataPath = '%s/data' % (self.extractedPath) - logging.CyberCPLogFileWriter.writeToFile('Data path: %s' % (self.dataPath)) self.databasesPath = '%s/databases' % (self.extractedPath) - logging.CyberCPLogFileWriter.writeToFile('Databases path: %s' % (self.databasesPath)) self.emailsPath = '%s/emails' % (self.extractedPath) - logging.CyberCPLogFileWriter.writeToFile('Emails path: %s' % (self.emailsPath)) ## Data diff --git a/s3Backups/s3Backups.py b/s3Backups/s3Backups.py index d2cb0510d..97c5ed102 100755 --- a/s3Backups/s3Backups.py +++ b/s3Backups/s3Backups.py @@ -97,43 +97,6 @@ class S3Backups(multi.Thread): json_data = json_data + ']' return json_data - def setupCron(self): - try: - - command = "sudo cat /etc/crontab" - crons = ProcessUtilities.outputExecutioner(command).splitlines() - - cronCheck = 1 - - for items in crons: - if items.find('s3Backups.py') > -1: - cronCheck = 0 - - tempPath = '/home/cyberpanel/' + str(randint(10000, 99999)) - - writeToFile = open(tempPath, "w") - - for items in crons: - writeToFile.writelines(items + "\n") - - if cronCheck: - writeToFile.writelines("0 0 * * * root /usr/local/CyberCP/bin/python /usr/local/CyberCP/s3Backups/s3Backups.py > /home/cyberpanel/error-logs.txt 2>&1\n") - - writeToFile.close() - - command = 'sudo mv ' + tempPath + " /etc/crontab" - ProcessUtilities.executioner(command) - - command = 'chown root:root /etc/crontab' - ProcessUtilities.executioner(command) - - try: - os.remove(tempPath) - except: - pass - except BaseException as msg: - logging.writeToFile(str(msg) + " [S3Backups.setupCron]") - def connectAccount(self): try: @@ -160,8 +123,6 @@ class S3Backups(multi.Thread): ## - self.setupCron() - return proc.ajax(1, None) except BaseException as msg: @@ -519,111 +480,6 @@ class S3Backups(multi.Thread): items.delete() return 0, status - def forceRunAWSBackup(self): - try: - - plan = BackupPlan.objects.get(name=self.data['planName']) - logging.writeToFile(plan.config) - bucketName = plan.bucket.strip('\n').strip(' ') - runTime = time.strftime("%d:%m:%Y") - - aws_access_key_id, aws_secret_access_key, region = self.fetchAWSKeys() - - client = boto3.client( - 's3', - aws_access_key_id = aws_access_key_id, - aws_secret_access_key = aws_secret_access_key, - #region_name=region - ) - - - config = TransferConfig(multipart_threshold=1024 * 25, max_concurrency=10, - multipart_chunksize=1024 * 25, use_threads=True) - - ## Set Expiration for objects - try: - - client.put_bucket_lifecycle_configuration( - Bucket='string', - LifecycleConfiguration={ - 'Rules': [ - { - 'Expiration': { - 'Days': plan.retention, - 'ExpiredObjectDeleteMarker': True - }, - 'ID': plan.name, - 'Prefix': '', - 'Filter': { - 'Prefix': plan.name + '/', - }, - 'Status': 'Enabled', - - }, - ] - } - ) - except BaseException as msg: - BackupLogs(owner=plan, timeStamp=time.strftime("%b %d %Y, %H:%M:%S"), level='ERROR', - msg=str(msg)).save() - - ## - - userID = self.request.session['userID'] - currentACL = ACLManager.loadedACL(userID) - - if currentACL['admin'] == 0: - BackupLogs(owner=plan, timeStamp=time.strftime("%b %d %Y, %H:%M:%S"), level='INFO', - msg='Unauthorised user tried to run AWS Backups.').save() - return 0 - - BackupLogs(owner=plan, level='INFO', timeStamp=time.strftime("%b %d %Y, %H:%M:%S"), - msg='Starting backup process..').save() - - PlanConfig = json.loads(plan.config) - - for items in plan.websitesinplan_set.all(): - - from plogical.backupUtilities import backupUtilities - tempStatusPath = "/home/cyberpanel/" + str(randint(1000, 9999)) - extraArgs = {} - extraArgs['domain'] = items.domain - extraArgs['tempStatusPath'] = tempStatusPath - extraArgs['data'] = PlanConfig['data'] - extraArgs['emails'] = PlanConfig['emails'] - extraArgs['databases'] = PlanConfig['databases'] - bu = backupUtilities(extraArgs) - result = bu.CloudBackups() - - finalResult = open(tempStatusPath, 'r').read() - - if result[0] == 1: - key = plan.name + '/' + runTime + '/' + result[1] - client.upload_file( - result[1], - bucketName, - key, - Config=config, - ) - - command = 'rm -f ' + result[1] - ProcessUtilities.executioner(command) - - BackupLogs(owner=plan, level='INFO', timeStamp=time.strftime("%b %d %Y, %H:%M:%S"), - msg='Backup successful for ' + items.domain + '.').save() - else: - BackupLogs(owner=plan, level='ERROR', timeStamp=time.strftime("%b %d %Y, %H:%M:%S"), - msg='Backup failed for ' + items.domain + '. Error: ' + finalResult).save() - - plan.lastRun = runTime - plan.save() - - BackupLogs(owner=plan, level='INFO', timeStamp=time.strftime("%b %d %Y, %H:%M:%S"), - msg='Backup Process Finished.').save() - except BaseException as msg: - logging.writeToFile(str(msg) + ' [S3Backups.runBackupPlan]') - plan = BackupPlan.objects.get(name=self.data['planName']) - BackupLogs(owner=plan, timeStamp=time.strftime("%b %d %Y, %H:%M:%S"), level='ERROR', msg=str(msg)).save() def connectAccountDO(self): try: @@ -649,9 +505,6 @@ class S3Backups(multi.Thread): credFile.write(self.data['credData']) credFile.close() - ## - - self.setupCron() return proc.ajax(1, None) @@ -1023,8 +876,6 @@ class S3Backups(multi.Thread): secretKey=self.data['secretKey']) newNode.save() - self.setupCron() - return proc.ajax(1, None) except BaseException as msg: @@ -1361,93 +1212,3 @@ class S3Backups(multi.Thread): except BaseException as msg: proc = httpProc(self.request, None, None) return proc.ajax(0, str(msg)) - - def runAWSBackups(self): - try: - admin = Administrator.objects.get(userName='admin') - self.request.session['userID'] = admin.pk - - for plan in BackupPlan.objects.all(): - lastRunDay = plan.lastRun.split(':')[0] - lastRunMonth = plan.lastRun.split(':')[1] - - if plan.freq == 'Daily' and lastRunDay != time.strftime("%d"): - self.data = {} - self.data['planName'] = plan.name - self.forceRunAWSBackup() - else: - if lastRunMonth == time.strftime("%m"): - days = int(time.strftime("%d")) - int(lastRunDay) - if days >= 6: - self.data = {} - self.data['planName'] = plan.name - self.forceRunAWSBackup() - else: - days = 30 - int(lastRunDay) - days = days + int(time.strftime("%d")) - if days >= 6: - self.data = {} - self.data['planName'] = plan.name - self.forceRunAWSBackup() - - for plan in BackupPlanDO.objects.all(): - lastRunDay = plan.lastRun.split(':')[0] - lastRunMonth = plan.lastRun.split(':')[1] - - if plan.freq == 'Daily' and lastRunDay != time.strftime("%d"): - self.data = {} - self.data['planName'] = plan.name - self.forceRunAWSBackupDO() - else: - if lastRunMonth == time.strftime("%m"): - days = int(time.strftime("%d")) - int(lastRunDay) - if days >= 6: - self.data = {} - self.data['planName'] = plan.name - self.forceRunAWSBackupDO() - else: - days = 30 - int(lastRunDay) - days = days + int(time.strftime("%d")) - if days >= 6: - self.data = {} - self.data['planName'] = plan.name - self.forceRunAWSBackupDO() - - for plan in BackupPlanMINIO.objects.all(): - lastRunDay = plan.lastRun.split(':')[0] - lastRunMonth = plan.lastRun.split(':')[1] - - if plan.freq == 'Daily' and lastRunDay != time.strftime("%d"): - self.data = {} - self.data['planName'] = plan.name - self.forceRunAWSBackupMINIO() - else: - if lastRunMonth == time.strftime("%m"): - days = int(time.strftime("%d")) - int(lastRunDay) - if days >= 6: - self.data = {} - self.data['planName'] = plan.name - self.forceRunAWSBackupMINIO() - else: - days = 30 - int(lastRunDay) - days = days + int(time.strftime("%d")) - if days >= 6: - self.data = {} - self.data['planName'] = plan.name - self.forceRunAWSBackupMINIO() - - except BaseException as msg: - logging.writeToFile(str(msg) + ' [S3Backups.runAWSBackups]') - - -def main(): - pathToFile = "/home/cyberpanel/" + str(randint(1000, 9999)) - file = open(pathToFile, "w") - file.close() - - finalData = json.dumps({'randomFile': pathToFile}) - requests.post("https://localhost:8090/api/runAWSBackups", data=finalData, verify=False) - - -if __name__ == "__main__": - main() From 87e38b8dcf9a847782953767d2edf883beea3fe2 Mon Sep 17 00:00:00 2001 From: Usman Nasir Date: Tue, 8 Dec 2020 21:19:23 +0500 Subject: [PATCH 19/20] add config column --- cloudAPI/cloudManager.py | 18 +++++++++++++++++- plogical/upgrade.py | 5 +++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/cloudAPI/cloudManager.py b/cloudAPI/cloudManager.py index 5e6185bcb..bf6af3c2a 100755 --- a/cloudAPI/cloudManager.py +++ b/cloudAPI/cloudManager.py @@ -1651,8 +1651,24 @@ class CloudManager: writeToFile.write('Starting..,0') writeToFile.close() + try: + data = str(int(self.data['data'])) + except: + data = '0' + + try: + emails = str(int(self.data['emails'])) + except: + emails = '0' + + try: + databases = str(int(self.data['databases'])) + except: + databases = '0' + + execPath = "/usr/local/CyberCP/bin/python " + virtualHostUtilities.cyberPanel + "/plogical/backupUtilities.py" - execPath = execPath + " CloudBackup --backupDomain %s --data 1 --emails 1 --databases 1 --tempStoragePath %s" % (self.data['domain'], tempStatusPath) + execPath = execPath + " CloudBackup --backupDomain %s --data %s --emails %s --databases %s --tempStoragePath %s" % (self.data['domain'], data, emails, databases, tempStatusPath) ProcessUtilities.popenExecutioner(execPath) final_dic = {'status': 1, 'tempStatusPath': tempStatusPath} diff --git a/plogical/upgrade.py b/plogical/upgrade.py index 9ff26da23..5ddd6d4d2 100755 --- a/plogical/upgrade.py +++ b/plogical/upgrade.py @@ -765,6 +765,11 @@ imap_folder_list_limit = 0 except: pass + try: + cursor.execute('ALTER TABLE s3Backups_backupplan ADD config longtext') + except: + pass + query = """CREATE TABLE `s3Backups_websitesinplan` ( `id` int(11) NOT NULL AUTO_INCREMENT, `domain` varchar(100) NOT NULL, From 081306e3246fa5c76be3349b10d20eda532e04f3 Mon Sep 17 00:00:00 2001 From: Usman Nasir Date: Tue, 8 Dec 2020 21:28:28 +0500 Subject: [PATCH 20/20] setup auto deletion for aws backups --- plogical/IncScheduler.py | 79 ++++++---------------------------------- 1 file changed, 11 insertions(+), 68 deletions(-) diff --git a/plogical/IncScheduler.py b/plogical/IncScheduler.py index 3915eabdd..055408bf7 100644 --- a/plogical/IncScheduler.py +++ b/plogical/IncScheduler.py @@ -686,12 +686,15 @@ Automatic backup failed for %s on %s. ts = time.time() + retentionSeconds = 86400 * plan.retention + for bucket in s3.buckets.all(): - for file in bucket.objects.all(): - result = float(ts - file.last_modified.timestamp()) - if result > 100.0: - file.delete() - print(result) + if bucket.name == plan.bucket: + for file in bucket.objects.all(): + result = float(ts - file.last_modified.timestamp()) + if result > retentionSeconds: + file.delete() + break except BaseException as msg: logging.writeToFile(str(msg) + ' [S3Backups.runBackupPlan]') @@ -699,71 +702,11 @@ Automatic backup failed for %s on %s. BackupLogs(owner=plan, timeStamp=time.strftime("%b %d %Y, %H:%M:%S"), level='ERROR', msg=str(msg)).save() @staticmethod - def runAWSBackups(): + def runAWSBackups(freq): try: for plan in BackupPlan.objects.all(): - lastRunDay = plan.lastRun.split(':')[0] - lastRunMonth = plan.lastRun.split(':')[1] - - if plan.freq == 'Daily' and lastRunDay != time.strftime("%d"): + if plan.freq == 'Daily' == freq: IncScheduler.forceRunAWSBackup(plan.name) - else: - if lastRunMonth == time.strftime("%m"): - days = int(time.strftime("%d")) - int(lastRunDay) - if days >= 6: - IncScheduler.forceRunAWSBackup(plan.name) - else: - days = 30 - int(lastRunDay) - days = days + int(time.strftime("%d")) - if days >= 6: - IncScheduler.forceRunAWSBackup(plan.name) - - # for plan in BackupPlanDO.objects.all(): - # lastRunDay = plan.lastRun.split(':')[0] - # lastRunMonth = plan.lastRun.split(':')[1] - # - # if plan.freq == 'Daily' and lastRunDay != time.strftime("%d"): - # self.data = {} - # self.data['planName'] = plan.name - # self.forceRunAWSBackupDO() - # else: - # if lastRunMonth == time.strftime("%m"): - # days = int(time.strftime("%d")) - int(lastRunDay) - # if days >= 6: - # self.data = {} - # self.data['planName'] = plan.name - # self.forceRunAWSBackupDO() - # else: - # days = 30 - int(lastRunDay) - # days = days + int(time.strftime("%d")) - # if days >= 6: - # self.data = {} - # self.data['planName'] = plan.name - # self.forceRunAWSBackupDO() - # - # for plan in BackupPlanMINIO.objects.all(): - # lastRunDay = plan.lastRun.split(':')[0] - # lastRunMonth = plan.lastRun.split(':')[1] - # - # if plan.freq == 'Daily' and lastRunDay != time.strftime("%d"): - # self.data = {} - # self.data['planName'] = plan.name - # self.forceRunAWSBackupMINIO() - # else: - # if lastRunMonth == time.strftime("%m"): - # days = int(time.strftime("%d")) - int(lastRunDay) - # if days >= 6: - # self.data = {} - # self.data['planName'] = plan.name - # self.forceRunAWSBackupMINIO() - # else: - # days = 30 - int(lastRunDay) - # days = days + int(time.strftime("%d")) - # if days >= 6: - # self.data = {} - # self.data['planName'] = plan.name - # self.forceRunAWSBackupMINIO() - except BaseException as msg: logging.writeToFile(str(msg) + ' [S3Backups.runAWSBackups]') @@ -785,7 +728,7 @@ def main(): IncScheduler.git(args.function) IncScheduler.checkDiskUsage() IncScheduler.startNormalBackups(args.function) - IncScheduler.runAWSBackups() + IncScheduler.runAWSBackups(args.function) if __name__ == "__main__":