From 65f070ba065db731804eb94c8e62f75bb24754b5 Mon Sep 17 00:00:00 2001 From: usmannasir <01-134132-158@student.bahria.edu.pk> Date: Fri, 11 May 2018 00:03:26 +0500 Subject: [PATCH] Bug fixes to Remote Backups. --- api/views.py | 16 +- backup/views.py | 5 +- plogical/backupUtilities.py | 142 +++++++++++++----- plogical/remoteTransferUtilities.py | 18 ++- .../websiteFunctions/createWebsite.html | 3 - .../templates/websiteFunctions/website.html | 3 - 6 files changed, 130 insertions(+), 57 deletions(-) diff --git a/api/views.py b/api/views.py index 8780150a6..88e9f80a1 100644 --- a/api/views.py +++ b/api/views.py @@ -23,6 +23,7 @@ import subprocess import shlex import re from dns.models import Domains,Records +from plogical.mailUtilities import mailUtilities # Create your views here. @@ -383,7 +384,6 @@ def fetchSSHkey(request): json_data = json.dumps(data_ret) return HttpResponse(json_data) - except BaseException, msg: data = {'pubKeyStatus': 0,'error_message': str(msg)} json_data = json.dumps(data) @@ -405,10 +405,19 @@ def remoteTransfer(request): dir = str(randint(1000, 9999)) ## - accountsToTransfer = ','.join(accountsToTransfer) + + mailUtilities.checkHome() + path = "/home/cyberpanel/accounts-" + str(randint(1000, 9999)) + writeToFile = open(path,'w') + + for items in accountsToTransfer: + writeToFile.writelines(items + "\n") + writeToFile.close() + + ## Accounts to transfer is a path to file, containing accounts. execPath = "sudo python " + virtualHostUtilities.cyberPanel + "/plogical/remoteTransferUtilities.py" - execPath = execPath + " remoteTransfer --ipAddress " + ipAddress + " --dir " + dir + " --accountsToTransfer " + accountsToTransfer + execPath = execPath + " remoteTransfer --ipAddress " + ipAddress + " --dir " + dir + " --accountsToTransfer " + path subprocess.Popen(shlex.split(execPath)) return HttpResponse(json.dumps({"transferStatus": 1, "dir": dir})) @@ -474,6 +483,7 @@ def FetchRemoteTransferStatus(request): data = json.loads(request.body) username = data['username'] password = data['password'] + dir = "/home/backup/transfer-"+str(data['dir'])+"/backup_log" try: diff --git a/backup/views.py b/backup/views.py index 878063915..fa5bc94bf 100644 --- a/backup/views.py +++ b/backup/views.py @@ -27,6 +27,7 @@ from xml.dom import minidom from dns.models import Domains,Records from mailServer.models import Domains as eDomains from mailServer.models import EUsers +from plogical.mailUtilities import mailUtilities @@ -1278,7 +1279,7 @@ def submitRemoteBackups(request): pubKey = data["pubKey"].strip("\n") else: final_json = json.dumps({'status': 0, - 'error_message': "I am sorry, I could not fetch key from remote server. Error Message: " +data['error_message'] + 'error_message': "I am sorry, I could not fetch key from remote server. Error Message: " + data['error_message'] }) return HttpResponse(final_json) @@ -1286,6 +1287,8 @@ def submitRemoteBackups(request): ## Writing key to a temporary location, to be read later by backup process. + mailUtilities.checkHome() + pathToKey = "/home/cyberpanel/" + str(randint(1000, 9999)) vhost = open(pathToKey, "w") diff --git a/plogical/backupUtilities.py b/plogical/backupUtilities.py index d533bbf10..769297c29 100644 --- a/plogical/backupUtilities.py +++ b/plogical/backupUtilities.py @@ -33,7 +33,7 @@ class backupUtilities: def startBackup(tempStoragePath,backupName,backupPath): try: - ## writing the name of backup file + ##### Writing the name of backup file. ## /home/example.com/backup/backupFileName backupFileNamePath = os.path.join(backupPath,"backupFileName") @@ -41,24 +41,25 @@ class backupUtilities: status.write(backupName) status.close() + ##### status = open(os.path.join(backupPath,'status'),"w") - status.write("Making archive of home directory\n") + status.write("Making archive of home directory.\n") status.close() - ## Parsing XML Meta file! + ##### Parsing XML Meta file! ## /home/example.com/backup/backup-example-06-50-03-Thu-Feb-2018 -- tempStoragePath backupMetaData = ElementTree.parse(os.path.join(tempStoragePath,'meta.xml')) - - ## Making archive of home directory + ##### Making archive of home directory domainName = backupMetaData.find('masterDomain').text ## /home/example.com/backup/backup-example-06-50-03-Thu-Feb-2018 -- tempStoragePath ## shutil.make_archive + make_archive(os.path.join(tempStoragePath,"public_html"), 'gztar', os.path.join("/home",domainName,"public_html")) ## backup email accounts @@ -76,6 +77,7 @@ class backupUtilities: databases = backupMetaData.findall('Databases/database') for database in databases: + dbName = database.find('dbName').text status = open(os.path.join(backupPath,'status'), "w") @@ -84,6 +86,8 @@ class backupUtilities: if mysqlUtilities.mysqlUtilities.createDatabaseBackup(dbName, tempStoragePath) == 0: raise BaseException + ##### Saving SSL Certificates if any + try: pathToStoreSSL = sslUtilities.Server_root + "/conf/vhosts/" + "SSL-" + domainName if os.path.exists(pathToStoreSSL): @@ -95,13 +99,41 @@ class backupUtilities: except BaseException, msg: logging.CyberCPLogFileWriter.writeToFile(str(msg) + " [startBackup]") - ## shutil.make_archive, ## shutil. + + ## Child Domains SSL. + + + childDomains = backupMetaData.findall('ChildDomains/domain') + + try: + for childDomain in childDomains: + + actualChildDomain = childDomain.find('domain').text + + pathToStoreSSL = sslUtilities.Server_root + "/conf/vhosts/" + "SSL-" + actualChildDomain + + if os.path.exists(pathToStoreSSL): + pathToStoreSSLPrivKey = pathToStoreSSL + "/privkey.pem" + pathToStoreSSLFullChain = pathToStoreSSL + "/fullchain.pem" + + tempKeyPath = os.path.join(tempStoragePath, actualChildDomain) + + if not os.path.exists(tempKeyPath): + os.mkdir(tempKeyPath) + + copy(pathToStoreSSLPrivKey, tempKeyPath + "/privkey.pem") + copy(pathToStoreSSLFullChain, tempKeyPath + "/fullchain.pem") + + except BaseException, msg: + logging.CyberCPLogFileWriter.writeToFile(str(msg) + " [startBackup]") + + ##### Saving SSL Certificates if any + + ## shutil.make_archive. Creating final package. + make_archive(os.path.join(backupPath,backupName), 'gztar', tempStoragePath) rmtree(tempStoragePath) - ## Saving SSL Certificates if any - - status = open(os.path.join(backupPath,'status'), "w") status.write("Completed\n") @@ -182,12 +214,38 @@ class backupUtilities: ########### Creating website and its dabases + ## extracting master domain for later use + backupMetaData = ElementTree.parse(os.path.join(completPath, "meta.xml")) + masterDomain = backupMetaData.find('masterDomain').text + try: finalData = json.dumps({'backupFile': backupName,"dir":dir}) r = requests.post("http://localhost:5003/websites/CreateWebsiteFromBackup", data=finalData,verify=False) data = json.loads(r.text) if data['createWebSiteStatus'] == 1: + + ## Let us try to restore SSL. + + if os.path.exists(completPath + "/privkey.pem"): + + pathToStoreSSL = sslUtilities.Server_root + "/conf/vhosts/" + "SSL-" + masterDomain + + if not os.path.exists(pathToStoreSSL): + os.mkdir(pathToStoreSSL) + + sslUtilities.installSSLForDomain(masterDomain) + + + pathToStoreSSLPrivKey = pathToStoreSSL + "/privkey.pem" + pathToStoreSSLFullChain = pathToStoreSSL + "/fullchain.pem" + + copy(completPath + "/privkey.pem", pathToStoreSSLPrivKey) + copy(completPath + "/fullchain.pem", pathToStoreSSLFullChain) + + command = "chown -R " + "lsadm" + ":" + "lsadm" + " " + pathToStoreSSL + cmd = shlex.split(command) + pass else: status = open(os.path.join(completPath,'status'), "w") @@ -201,38 +259,22 @@ class backupUtilities: logging.CyberCPLogFileWriter.writeToFile(str(msg) + " [startRestore]") return 0 - ###########Ccreating child/sub/addon/parked domains + ########### Creating child/sub/addon/parked domains status = open(os.path.join(completPath,'status'), "w") status.write("Creating Child Domains!") status.close() - ## reading meta file to create subdomains + ## Reading meta file to create subdomains - backupMetaData = ElementTree.parse(os.path.join(completPath,"meta.xml")) - - ## extracting master domain for later use - masterDomain = backupMetaData.find('masterDomain').text externalApp = backupMetaData.find('externalApp').text websiteHome = os.path.join("/home",masterDomain,"public_html") + + ### Restoring Child Domains if any. + childDomains = backupMetaData.findall('ChildDomains/domain') - ## Let us try to restore SSL. - - if os.path.exists(completPath + "/privkey.pem"): - sslUtilities.installSSLForDomain(masterDomain) - - pathToStoreSSL = sslUtilities.Server_root + "/conf/vhosts/" + "SSL-" + masterDomain - pathToStoreSSLPrivKey = pathToStoreSSL + "/privkey.pem" - pathToStoreSSLFullChain = pathToStoreSSL + "/fullchain.pem" - - copy(completPath + "/privkey.pem", pathToStoreSSLPrivKey) - copy(completPath + "/fullchain.pem", pathToStoreSSLFullChain) - - command = "chown " + "lsadm" + ":" + "lsadm" + " " + pathToStoreSSL - cmd = shlex.split(command) - try: for childDomain in childDomains: @@ -252,6 +294,29 @@ class backupUtilities: if data['createWebSiteStatus'] == 1: rmtree(websiteHome) + + ## Let us try to restore SSL for Child Domains. + + tempPath = os.path.join(completPath, domain) + + if os.path.exists(tempPath + "/privkey.pem"): + + pathToStoreSSL = sslUtilities.Server_root + "/conf/vhosts/" + "SSL-" + domain + + if not os.path.exists(pathToStoreSSL): + os.mkdir(pathToStoreSSL) + + sslUtilities.installSSLForDomain(domain) + + pathToStoreSSLPrivKey = pathToStoreSSL + "/privkey.pem" + pathToStoreSSLFullChain = pathToStoreSSL + "/fullchain.pem" + + copy(tempPath + "/privkey.pem", pathToStoreSSLPrivKey) + copy(tempPath + "/fullchain.pem", pathToStoreSSLFullChain) + + command = "chown -R " + "lsadm" + ":" + "lsadm" + " " + pathToStoreSSL + cmd = shlex.split(command) + continue else: status = open(os.path.join(completPath,'status'), "w") @@ -267,20 +332,23 @@ class backupUtilities: logging.CyberCPLogFileWriter.writeToFile(str(msg) + " [startRestore]") return 0 - ## Restoring email accounts + ## Restore Aliases status = open(os.path.join(completPath, 'status'), "w") - status.write("Restoring email accounts!") + status.write("Restoring Domain Aliases!") status.close() - - ## Restore Aliases - aliases = backupMetaData.findall('Aliases/alias') for items in aliases: createAlias(masterDomain, items.text, 0, "", "") + ## Restoring email accounts + + status = open(os.path.join(completPath, 'status'), "w") + status.write("Restoring email accounts!") + status.close() + emailAccounts = backupMetaData.findall('emails/emailAccount') try: @@ -332,10 +400,6 @@ class backupUtilities: ## Databases restored - ## Restoring Aliases - - aliases = backupMetaData.findall('Databases/database') - status = open(os.path.join(completPath, 'status'), "w") status.write("Extracting web home data!") status.close() diff --git a/plogical/remoteTransferUtilities.py b/plogical/remoteTransferUtilities.py index c68508015..0922dd60d 100644 --- a/plogical/remoteTransferUtilities.py +++ b/plogical/remoteTransferUtilities.py @@ -47,17 +47,22 @@ class remoteTransferUtilities: except BaseException,msg: logging.CyberCPLogFileWriter.writeToFile("For remote transfer, I am not able to write key to auth file, Error Message: "+str(msg)) - print "0,"+"For remote transfer, I am not able to write key to auth file, Error Message: "+str(msg) + print "0,"+"For remote transfer, I am not able to write key to auth file, Error Message: " + str(msg) - ## house keeping function to run remote backups + ## House keeping function to run remote backups @staticmethod - def remoteTransfer(ipAddress,dir,accountsToTransfer): + def remoteTransfer(ipAddress, dir, accountsToTransfer): try: destination = "/home/backup/transfer-" + dir backupLogPath = destination + "/backup_log" - accountsToTransfer = accountsToTransfer.split(",") + data = open(accountsToTransfer, 'r').readlines() + + accountsToTransfer = [] + + for items in data: + accountsToTransfer.append(items.strip('\n')) if not os.path.exists(destination): os.makedirs(destination) @@ -108,7 +113,7 @@ class remoteTransferUtilities: writeToFile.writelines(str(msg) + " [5010]" + "\n") writeToFile.close() logging.CyberCPLogFileWriter.writeToFile(str(msg) + " [remoteTransfer]") - return [0, msg] + return [0, str(msg)] ## destination = /home/backup/transfer-2558 @@ -125,9 +130,6 @@ class remoteTransferUtilities: try: - if virtualHost == "vmail" or virtualHost == "backup": - continue - writeToFile = open(backupLogPath, "a") writeToFile.writelines("[" + time.strftime( "%I-%M-%S-%a-%b-%Y") + "]" + " Currently generating local backups for: " + virtualHost + "\n") diff --git a/websiteFunctions/templates/websiteFunctions/createWebsite.html b/websiteFunctions/templates/websiteFunctions/createWebsite.html index 2c5d7f19e..afdf9e12d 100644 --- a/websiteFunctions/templates/websiteFunctions/createWebsite.html +++ b/websiteFunctions/templates/websiteFunctions/createWebsite.html @@ -88,9 +88,6 @@ diff --git a/websiteFunctions/templates/websiteFunctions/website.html b/websiteFunctions/templates/websiteFunctions/website.html index a2bb97e91..ed90483cf 100644 --- a/websiteFunctions/templates/websiteFunctions/website.html +++ b/websiteFunctions/templates/websiteFunctions/website.html @@ -353,9 +353,6 @@