From ba8b63df6f8abcaec5e683a048712a6564c03845 Mon Sep 17 00:00:00 2001 From: "usman@cyberpersons.com" Date: Thu, 9 Mar 2023 21:09:13 +0500 Subject: [PATCH] use restic merge for combining backups from https://community.cyberpanel.net/t/cyberpanel-backups-v2-proposal/40370/95?u=usmannasir --- plogical/Backupsv2.py | 84 +++++++++++++++++++------------------- plogical/mysqlUtilities.py | 4 +- 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/plogical/Backupsv2.py b/plogical/Backupsv2.py index 88e0990c1..1a80e6f12 100644 --- a/plogical/Backupsv2.py +++ b/plogical/Backupsv2.py @@ -123,7 +123,7 @@ pass = {ObsecurePassword} command = f'chown {self.website.externalApp}:{self.website.externalApp} {self.FinalPathRuctic}/config.json' ProcessUtilities.executioner(command) - command = f'rustic -r {self.repo} backup {self.FinalPathRuctic}/config.json --password ""' + command = f'rustic -r {self.repo} backup {self.FinalPathRuctic}/config.json --json --password "" 2>/dev/null' result = json.loads(ProcessUtilities.outputExecutioner(command, self.website.externalApp, True).rstrip('\n')) try: @@ -147,12 +147,11 @@ pass = {ObsecurePassword} snapshots= f'{snapshots} {snapshot}' - - command = f'rustic -r {self.repo} merge {snapshots} --password ""' + command = f'rustic -r {self.repo} merge {snapshots} --password "" --json 2>/dev/null' result = json.loads(ProcessUtilities.outputExecutioner(command, self.website.externalApp, True).rstrip('\n')) command = f'rustic -r {self.repo} forget {snapshots} --password ""' - result = json.loads(ProcessUtilities.outputExecutioner(command, self.website.externalApp, True).rstrip('\n')) + result = ProcessUtilities.outputExecutioner(command, self.website.externalApp, True) def InitiateBackup(self): @@ -360,11 +359,11 @@ pass = {ObsecurePassword} return 0 self.UpdateStatus('Website data backup completed successfully..,70', CPBackupsV2.RUNNING) - if self.data['BackupEmails']: - self.UpdateStatus('Backing up emails..,75', CPBackupsV2.RUNNING) - if self.BackupEmailsRustic() == 0: - return 0 - self.UpdateStatus('Emails backup completed successfully..,85', CPBackupsV2.RUNNING) + # if self.data['BackupEmails']: + # self.UpdateStatus('Backing up emails..,75', CPBackupsV2.RUNNING) + # if self.BackupEmailsRustic() == 0: + # return 0 + # self.UpdateStatus('Emails backup completed successfully..,85', CPBackupsV2.RUNNING) ### Finally change the backup rustic folder to the website user owner @@ -452,49 +451,50 @@ pass = {ObsecurePassword} CurrentDBPath = f"{self.FinalPathRuctic}/{key}.sql" - DBResult = mysqlUtilities.createDatabaseBackup(key, self.FinalPathRuctic) + DBResult, SnapID = mysqlUtilities.createDatabaseBackup(key, self.FinalPathRuctic, 1, self.repo, self.website.externalApp) if DBResult == 1: + self.snapshots.append(SnapID) - command = f'chown {self.website.externalApp}:{self.website.externalApp} {CurrentDBPath}' - ProcessUtilities.executioner(command) + #command = f'chown {self.website.externalApp}:{self.website.externalApp} {CurrentDBPath}' + #ProcessUtilities.executioner(command) ## Now pack config into same thing #command = f'chown {self.website.externalApp}:{self.website.externalApp} {self.FinalPathRuctic}/config.json' #ProcessUtilities.executioner(command) - command = f'rustic -r {self.repo} backup {CurrentDBPath} --password "" --json 2>/dev/null' - print(f'db command rustic: {command}') - result = json.loads( - ProcessUtilities.outputExecutioner(command, self.website.externalApp, True).rstrip('\n')) - - try: - SnapShotID = result['id'] ## snapshot id that we need to store in db - files_new = result['summary']['files_new'] ## basically new files in backup - total_duration = result['summary']['total_duration'] ## time taken - - self.snapshots.append(SnapShotID) - - ### Config is saved with each database, snapshot of config is attached to db snapshot with parent - - #self.BackupConfig(SnapShotID) - - command = f'chown cyberpanel:cyberpanel {self.FinalPathRuctic}' - ProcessUtilities.executioner(command) - - except BaseException as msg: - self.UpdateStatus(f'Backup failed as no snapshot id found, error: {str(msg)}', - CPBackupsV2.FAILED) - return 0 - - - for dbUsers in value: - print(f'User: {dbUsers["user"]}, Host: {dbUsers["host"]}, Pass: {dbUsers["password"]}') - - command = f'rm -f {CurrentDBPath}' - ProcessUtilities.executioner(command) + # command = f'rustic -r {self.repo} backup {CurrentDBPath} --password "" --json 2>/dev/null' + # print(f'db command rustic: {command}') + # result = json.loads( + # ProcessUtilities.outputExecutioner(command, self.website.externalApp, True).rstrip('\n')) + # + # try: + # SnapShotID = result['id'] ## snapshot id that we need to store in db + # files_new = result['summary']['files_new'] ## basically new files in backup + # total_duration = result['summary']['total_duration'] ## time taken + # + # self.snapshots.append(SnapShotID) + # + # ### Config is saved with each database, snapshot of config is attached to db snapshot with parent + # + # #self.BackupConfig(SnapShotID) + # + # command = f'chown cyberpanel:cyberpanel {self.FinalPathRuctic}' + # ProcessUtilities.executioner(command) + # + # except BaseException as msg: + # self.UpdateStatus(f'Backup failed as no snapshot id found, error: {str(msg)}', + # CPBackupsV2.FAILED) + # return 0 + # + # + # for dbUsers in value: + # print(f'User: {dbUsers["user"]}, Host: {dbUsers["host"]}, Pass: {dbUsers["password"]}') + # + # command = f'rm -f {CurrentDBPath}' + # ProcessUtilities.executioner(command) else: command = f'rm -f {CurrentDBPath}' diff --git a/plogical/mysqlUtilities.py b/plogical/mysqlUtilities.py index 62efd3f1c..2be4a3cec 100755 --- a/plogical/mysqlUtilities.py +++ b/plogical/mysqlUtilities.py @@ -245,7 +245,7 @@ class mysqlUtilities: return str(msg) @staticmethod - def createDatabaseBackup(databaseName, tempStoragePath, rustic=0, RusticRepoName = None): + def createDatabaseBackup(databaseName, tempStoragePath, rustic=0, RusticRepoName = None, externalApp = None): try: passFile = "/etc/cyberpanel/mysqlPassword" @@ -313,7 +313,7 @@ password=%s else: SHELL = True - command = f'mysqldump --defaults-extra-file=/home/cyberpanel/.my.cnf -u {mysqluser} --host={mysqlhost} --port {mysqlport} --add-drop-table --allow-keywords --complete-insert --quote-names --skip-comments {databaseName} | rustic -r {RusticRepoName} backup --stdin-filename {databaseName}.sql --password "" --json 2>/dev/null' + command = f'mysqldump --defaults-extra-file=/home/cyberpanel/.my.cnf -u {mysqluser} --host={mysqlhost} --port {mysqlport} --add-drop-table --allow-keywords --complete-insert --quote-names --skip-comments {databaseName} | sudo -u {externalApp} rustic -r {RusticRepoName} backup --stdin-filename {databaseName}.sql - --password "" --json 2>/dev/null' if os.path.exists(ProcessUtilities.debugPath): logging.CyberCPLogFileWriter.writeToFile(command)