From fdf7f9124cc2b19c0d3a43dd2c84d6af65321a7f Mon Sep 17 00:00:00 2001 From: Usman Nasir Date: Wed, 11 Dec 2019 10:11:20 +0500 Subject: [PATCH 1/5] wait time for terminal --- WebTerminal/CPWebSocket.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WebTerminal/CPWebSocket.py b/WebTerminal/CPWebSocket.py index c5630561b..b378f959e 100644 --- a/WebTerminal/CPWebSocket.py +++ b/WebTerminal/CPWebSocket.py @@ -79,7 +79,7 @@ class SSHServer(multi.Thread): else: self.websocket.sendMessage(self.shell.recv(9000).decode("utf-8")) else: - pass + time.sleep(0.01) else: return 0 except BaseException, msg: From 2fdf0a9915e81545a6e4a860a24bc0a466384a14 Mon Sep 17 00:00:00 2001 From: Usman Nasir Date: Wed, 11 Dec 2019 10:20:06 +0500 Subject: [PATCH 2/5] bug fix: bypass ipv6 --- CyberCP/secMiddleware.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CyberCP/secMiddleware.py b/CyberCP/secMiddleware.py index 3efed4d2b..fa382901c 100755 --- a/CyberCP/secMiddleware.py +++ b/CyberCP/secMiddleware.py @@ -74,7 +74,7 @@ class secMiddleware: if request.build_absolute_uri().find('saveSpamAssassinConfigurations') > -1 or request.build_absolute_uri().find('docker') > -1 or request.build_absolute_uri().find('cloudAPI') > -1 or request.build_absolute_uri().find('filemanager') > -1 or request.build_absolute_uri().find('verifyLogin') > -1 or request.build_absolute_uri().find('submitUserCreation') > -1: continue - if key == 'backupDestinations' or key == 'ports' or key == 'imageByPass' or key == 'passwordByPass' or key == 'cronCommand' or key == 'emailMessage' or key == 'configData' or key == 'rewriteRules' or key == 'modSecRules' or key == 'recordContentTXT' or key == 'SecAuditLogRelevantStatus' or key == 'fileContent': + if key == 'recordContentAAAA' or key == 'backupDestinations' or key == 'ports' or key == 'imageByPass' or key == 'passwordByPass' or key == 'cronCommand' or key == 'emailMessage' or key == 'configData' or key == 'rewriteRules' or key == 'modSecRules' or key == 'recordContentTXT' or key == 'SecAuditLogRelevantStatus' or key == 'fileContent': continue if value.find(';') > -1 or value.find('&&') > -1 or value.find('|') > -1 or value.find('...') > -1 \ or value.find("`") > -1 or value.find("$") > -1 or value.find("(") > -1 or value.find(")") > -1 \ From 52e7f038f1fb3ea1c37afa5ef12ccd9581c3527a Mon Sep 17 00:00:00 2001 From: Usman Nasir Date: Fri, 13 Dec 2019 17:43:00 +0500 Subject: [PATCH 3/5] bug fix: install --- install/install.py | 2 +- install/installCyberPanel.py | 2 +- plogical/backupSchedule.py | 3 +++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/install/install.py b/install/install.py index ccbbabe0c..5b07ac2f2 100755 --- a/install/install.py +++ b/install/install.py @@ -1796,7 +1796,7 @@ imap_folder_list_limit = 0 preFlightsChecks.call(command, self.distro, command, command, 1, 0, os.EX_OSERR) if self.distro == ubuntu: - command = 'apt install opendkim-tools' + command = 'apt install opendkim-tools -y' preFlightsChecks.call(command, self.distro, command, command, 1, 0, os.EX_OSERR) command = 'mkdir -p /etc/opendkim/keys/' diff --git a/install/installCyberPanel.py b/install/installCyberPanel.py index 5c3e76d2c..c72a5c461 100755 --- a/install/installCyberPanel.py +++ b/install/installCyberPanel.py @@ -455,7 +455,7 @@ class InstallCyberPanel: try: - f = open('/etc/resolv.conf', 'w') + f = open('/etc/resolv.conf', 'a') f.write('nameserver 8.8.8.8') f.close() except IOError as e: diff --git a/plogical/backupSchedule.py b/plogical/backupSchedule.py index eb8b3a482..a739fb780 100755 --- a/plogical/backupSchedule.py +++ b/plogical/backupSchedule.py @@ -16,6 +16,7 @@ from websiteFunctions.models import Websites, Backups from plogical.processUtilities import ProcessUtilities from random import randint import json, requests +from datetime import datetime class backupSchedule: @@ -34,6 +35,8 @@ class backupSchedule: def createLocalBackup(virtualHost, backupLogPath): try: + startingTime = datetime.now() + backupSchedule.remoteBackupLogging(backupLogPath, "Starting local backup for: " + virtualHost) ### From 08cc518b1adbca0071340a38a7254f89373a18db Mon Sep 17 00:00:00 2001 From: Usman Nasir Date: Fri, 13 Dec 2019 22:18:53 +0500 Subject: [PATCH 4/5] bug fix: add check to see if backup process killed --- install/installCyberPanel.py | 16 ++++++------ plogical/backupSchedule.py | 12 +++++++++ plogical/backupScheduleLocal.py | 21 ++++++++++------ plogical/backupUtilities.py | 43 +++++++++++++++++++++++++++++++++ 4 files changed, 76 insertions(+), 16 deletions(-) diff --git a/install/installCyberPanel.py b/install/installCyberPanel.py index c72a5c461..2a7de196b 100755 --- a/install/installCyberPanel.py +++ b/install/installCyberPanel.py @@ -454,14 +454,14 @@ class InstallCyberPanel: - try: - f = open('/etc/resolv.conf', 'a') - f.write('nameserver 8.8.8.8') - f.close() - except IOError as e: - InstallCyberPanel.stdOut("[ERROR] Unable to create /etc/resolv.conf: " + str(e) + - ". This may need to be fixed manually as 'echo \"nameserver 8.8.8.8\"> " - "/etc/resolv.conf'", 1, 1, os.EX_OSERR) + # try: + # f = open('/etc/resolv.conf', 'a') + # f.write('nameserver 8.8.8.8') + # f.close() + # except IOError as e: + # InstallCyberPanel.stdOut("[ERROR] Unable to create /etc/resolv.conf: " + str(e) + + # ". This may need to be fixed manually as 'echo \"nameserver 8.8.8.8\"> " + # "/etc/resolv.conf'", 1, 1, os.EX_OSERR) if self.distro == centos: command = 'yum -y install epel-release' diff --git a/plogical/backupSchedule.py b/plogical/backupSchedule.py index a739fb780..1e59a537c 100755 --- a/plogical/backupSchedule.py +++ b/plogical/backupSchedule.py @@ -52,8 +52,12 @@ class backupSchedule: tempStoragePath = data['tempStorage'] backupSchedule.remoteBackupLogging(backupLogPath, "Waiting for backup to complete.. ") + time.sleep(5) + schedulerPath = '/home/cyberpanel/%s-backup.txt' % (virtualHost) while (1): + diff = datetime.now() - startingTime + backupDomain = virtualHost status = os.path.join("/home", backupDomain, "backup/status") backupFileNamePath = os.path.join("/home", backupDomain, "backup/backupFileName") @@ -117,6 +121,14 @@ class backupSchedule: except: pass return 0, tempStoragePath + elif os.path.exists(schedulerPath): + os.remove(schedulerPath) + return 0, 'Backup process killed without reporting any error.' + + + + + except BaseException, msg: logging.CyberCPLogFileWriter.writeToFile(str(msg) + " [119:startBackup]") return 0, str(msg) diff --git a/plogical/backupScheduleLocal.py b/plogical/backupScheduleLocal.py index 7129b9867..d5ce1a6bf 100755 --- a/plogical/backupScheduleLocal.py +++ b/plogical/backupScheduleLocal.py @@ -24,17 +24,22 @@ class backupScheduleLocal: for virtualHost in os.listdir("/home"): if match(r'([\da-z\.-]+\.[a-z\.]{2,12}|[\d\.]+)([\/:?=&#]{1}[\da-z\.-]+)*[\/\?]?', virtualHost, M | I): - retValues = backupSchedule.createLocalBackup(virtualHost, backupLogPath) + try: + retValues = backupSchedule.createLocalBackup(virtualHost, backupLogPath) - if os.path.exists(backupScheduleLocal.localBackupPath): - backupPath = retValues[1] + ".tar.gz" - localBackupPath = '%s/%s' % (open(backupScheduleLocal.localBackupPath, 'r').read().rstrip('/'), time.strftime("%b-%d-%Y")) + if os.path.exists(backupScheduleLocal.localBackupPath): + backupPath = retValues[1] + ".tar.gz" + localBackupPath = '%s/%s' % (open(backupScheduleLocal.localBackupPath, 'r').read().rstrip('/'), time.strftime("%b-%d-%Y")) - command = 'mkdir -p %s' % (localBackupPath) - ProcessUtilities.normalExecutioner(command) + command = 'mkdir -p %s' % (localBackupPath) + ProcessUtilities.normalExecutioner(command) + + command = 'mv %s %s' % (backupPath, localBackupPath) + ProcessUtilities.normalExecutioner(command) + except BaseException, msg: + backupSchedule.remoteBackupLogging(backupLogPath, + '[ERROR] Backup failed for %s, error: %s moving on..' % (virtualHost, str(msg))) - command = 'mv %s %s' % (backupPath, localBackupPath) - ProcessUtilities.normalExecutioner(command) diff --git a/plogical/backupUtilities.py b/plogical/backupUtilities.py index b1132eac0..053de8859 100755 --- a/plogical/backupUtilities.py +++ b/plogical/backupUtilities.py @@ -263,6 +263,10 @@ class backupUtilities: ##### Writing the name of backup file. ## /home/example.com/backup/backupFileName + pidFile = '%sstartBackup' % (backupPath) + writeToFile = open(pidFile, 'w') + writeToFile.writelines(str(os.getpid())) + writeToFile.close() backupFileNamePath = os.path.join(backupPath,"backupFileName") logging.CyberCPLogFileWriter.statusWriter(backupFileNamePath, backupName) @@ -321,9 +325,17 @@ class backupUtilities: logging.CyberCPLogFileWriter.statusWriter(status, "Aborted, "+ str(msg) + ".[365] [5009]") print ("Aborted, "+ str(msg) + ".[365] [5009]") + os.remove(pidFile) + @staticmethod def BackupRoot(tempStoragePath, backupName, backupPath, metaPath=None): + pidFile = '%sBackupRoot' % (backupPath) + + writeToFile = open(pidFile, 'w') + writeToFile.writelines(str(os.getpid())) + writeToFile.close() + status = os.path.join(backupPath, 'status') metaPathInBackup = os.path.join(tempStoragePath, 'meta.xml') backupMetaData = ElementTree.parse(metaPathInBackup) @@ -422,6 +434,7 @@ class backupUtilities: items.save() logging.CyberCPLogFileWriter.statusWriter(status, "Completed\n") + os.remove(pidFile) @staticmethod def initiateBackup(tempStoragePath,backupName,backupPath): @@ -1056,16 +1069,24 @@ def submitBackupCreation(tempStoragePath, backupName, backupPath, backupDomain): ## + schedulerPath = '/home/cyberpanel/%s-backup.txt' % (backupDomain) + if not os.path.exists(backupPath) or not os.path.islink(backupPath): command = 'mkdir -p %s' % (backupPath) ProcessUtilities.executioner(command) else: + writeToFile = open(schedulerPath, 'w') + writeToFile.writelines('error') + writeToFile.close() return 0 if not os.path.exists(backupPath) or not os.path.islink(backupPath): command = 'chown -R %s:%s %s' % (website.externalApp, website.externalApp, backupPath) ProcessUtilities.executioner(command) else: + writeToFile = open(schedulerPath, 'w') + writeToFile.writelines('error') + writeToFile.close() return 0 ## @@ -1074,12 +1095,18 @@ def submitBackupCreation(tempStoragePath, backupName, backupPath, backupDomain): command = 'mkdir -p %s' % (tempStoragePath) ProcessUtilities.executioner(command) else: + writeToFile = open(schedulerPath, 'w') + writeToFile.writelines('error') + writeToFile.close() return 0 if not os.path.exists(tempStoragePath) or not os.path.islink(tempStoragePath): command = 'chown -R %s:%s %s' % (website.externalApp, website.externalApp, tempStoragePath) ProcessUtilities.executioner(command) else: + writeToFile = open(schedulerPath, 'w') + writeToFile.writelines('error') + writeToFile.close() return 0 ## @@ -1087,12 +1114,18 @@ def submitBackupCreation(tempStoragePath, backupName, backupPath, backupDomain): command = 'touch %s' % (status) ProcessUtilities.executioner(command) else: + writeToFile = open(schedulerPath, 'w') + writeToFile.writelines('error') + writeToFile.close() return 0 if not os.path.exists(status) or not os.path.islink(status): command = 'chown cyberpanel:cyberpanel %s' % (status) ProcessUtilities.executioner(command) else: + writeToFile = open(schedulerPath, 'w') + writeToFile.writelines('error') + writeToFile.close() return 0 @@ -1100,6 +1133,9 @@ def submitBackupCreation(tempStoragePath, backupName, backupPath, backupDomain): if result[0] == 0: + writeToFile = open(schedulerPath, 'w') + writeToFile.writelines('error') + writeToFile.close() logging.CyberCPLogFileWriter.statusWriter(status, str(result[1]) + ' [1084][5009]') return 0 @@ -1113,6 +1149,9 @@ def submitBackupCreation(tempStoragePath, backupName, backupPath, backupDomain): output = ProcessUtilities.outputExecutioner(execPath, website.externalApp) if output.find('[5009') > -1: logging.CyberCPLogFileWriter.writeToFile(output) + writeToFile = open(schedulerPath, 'w') + writeToFile.writelines('error') + writeToFile.close() return 0 ## Backing up databases @@ -1126,6 +1165,9 @@ def submitBackupCreation(tempStoragePath, backupName, backupPath, backupDomain): dbName = database.find('dbName').text if mysqlUtilities.mysqlUtilities.createDatabaseBackup(dbName, '/home/cyberpanel') == 0: + writeToFile = open(schedulerPath, 'w') + writeToFile.writelines('error') + writeToFile.close() return 0 command = 'mv /home/cyberpanel/%s.sql %s/%s.sql' % (dbName, tempStoragePath, dbName) @@ -1134,6 +1176,7 @@ def submitBackupCreation(tempStoragePath, backupName, backupPath, backupDomain): ## output = ProcessUtilities.outputExecutioner(execPath, website.externalApp) + if output.find('1,None') > -1: execPath = "sudo nice -n 10 /usr/local/CyberCP/bin/python2 " + virtualHostUtilities.cyberPanel + "/plogical/backupUtilities.py" execPath = execPath + " BackupRoot --tempStoragePath " + tempStoragePath + " --backupName " \ From 0cc6a429f90ae3520f1a4f5d7bcd63fa4526b931 Mon Sep 17 00:00:00 2001 From: Usman Nasir Date: Sat, 14 Dec 2019 16:23:47 +0500 Subject: [PATCH 5/5] bug fix: scheduled backups --- plogical/backupSchedule.py | 12 +++++++++++- plogical/backupScheduleLocal.py | 11 +++++++++++ test.sh | 1 + 3 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 test.sh diff --git a/plogical/backupSchedule.py b/plogical/backupSchedule.py index 1e59a537c..12a59ea91 100755 --- a/plogical/backupSchedule.py +++ b/plogical/backupSchedule.py @@ -17,9 +17,11 @@ from plogical.processUtilities import ProcessUtilities from random import randint import json, requests from datetime import datetime +import signal + class backupSchedule: - + now = datetime.now() @staticmethod def remoteBackupLogging(fileName, message): @@ -262,6 +264,14 @@ class backupSchedule: def main(): backupSchedule.prepare() +def handler(signum, frame): + diff = datetime.now() - backupSchedule.now + logging.CyberCPLogFileWriter.writeToFile('Signal: %s, time spent: %s' % (str(signum), str(diff.total_seconds()))) + if __name__ == "__main__": + for i in range(1,32): + if i == 9 or i == 19 or i == 32: + continue + signal.signal(i, handler) main() \ No newline at end of file diff --git a/plogical/backupScheduleLocal.py b/plogical/backupScheduleLocal.py index d5ce1a6bf..73f3d5ae7 100755 --- a/plogical/backupScheduleLocal.py +++ b/plogical/backupScheduleLocal.py @@ -4,9 +4,12 @@ import time from backupSchedule import backupSchedule from plogical.processUtilities import ProcessUtilities from re import match,I,M +import signal +from datetime import datetime class backupScheduleLocal: localBackupPath = '/home/cyberpanel/localBackupPath' + now = datetime.now() @staticmethod def prepare(): @@ -62,5 +65,13 @@ def main(): backupScheduleLocal.prepare() +def handler(signum, frame): + diff = datetime.now() - backupScheduleLocal.now + logging.CyberCPLogFileWriter.writeToFile('Signal: %s, time spent: %s' % (str(signum), str(diff.total_seconds()))) + if __name__ == "__main__": + for i in range(1,32): + if i == 9 or i == 19 or i == 32: + continue + signal.signal(i, handler) main() \ No newline at end of file diff --git a/test.sh b/test.sh new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/test.sh @@ -0,0 +1 @@ +