Merge pull request #834 from nickchomey/rclone

Rclone Incremental backups
This commit is contained in:
Usman Nasir
2022-03-20 22:49:56 +05:00
committed by GitHub
10 changed files with 111 additions and 138 deletions

View File

@@ -43,6 +43,7 @@ class IncJobs(multi.Thread):
self.website = ''
self.backupDestinations = ''
self.jobid = 0
self.jobids = ''
self.metaPath = ''
self.path = ''
self.reconstruct = ''
@@ -57,9 +58,9 @@ class IncJobs(multi.Thread):
self.restorePoint()
def getRemoteBackups(self):
if self.backupDestinations[:4] == 'sftp':
path = '/home/backup/%s' % (self.website)
command = 'export RESTIC_PASSWORD=%s PATH=${PATH}:/usr/bin && restic -r %s:%s snapshots' % (
if self.backupDestinations == 'onedrive':
path = '%s' % (self.website)
command = 'export RESTIC_PASSWORD=%s PATH=${PATH}:/usr/bin && restic -r rclone:%s:%s snapshots' % (
self.passwordFile, self.backupDestinations, path)
return ProcessUtilities.outputExecutioner(command, self.externalApp).split('\n')
else:
@@ -184,8 +185,8 @@ class IncJobs(multi.Thread):
key, secret = self.getAWSData()
command = 'export AWS_ACCESS_KEY_ID=%s AWS_SECRET_ACCESS_KEY=%s && restic -r s3:s3.amazonaws.com/%s forget %s --password-file %s' % (
key, secret, self.website, snapshotID, self.passwordFile)
command = 'export AWS_ACCESS_KEY_ID=%s AWS_SECRET_ACCESS_KEY=%s && restic -r s3:s3.amazonaws.com/%s forget %s --prune --password-file %s' % (
key, secret, self.website, self.jobids, self.passwordFile)
result = ProcessUtilities.outputExecutioner(command, self.externalApp)
@@ -194,11 +195,6 @@ class IncJobs(multi.Thread):
else:
logging.statusWriter(self.statusPath, 'Failed: %s. [5009]' % (result), 1)
return 0
command = 'export AWS_ACCESS_KEY_ID=%s AWS_SECRET_ACCESS_KEY=%s && restic -r s3:s3.amazonaws.com/%s prune --password-file %s' % (
key, secret, self.website, self.passwordFile)
ProcessUtilities.outputExecutioner(command, self.externalApp)
else:
self.backupDestinations = self.jobid.destination
@@ -254,7 +250,7 @@ class IncJobs(multi.Thread):
repoLocation = '/home/%s/incbackup' % (self.website)
command = 'restic -r %s forget %s --password-file %s' % (repoLocation, self.jobid.snapshotid, self.passwordFile)
command = 'restic -r %s forget %s --prune --password-file %s' % (repoLocation, self.jobids, self.passwordFile)
result = ProcessUtilities.outputExecutioner(command, self.externalApp)
if result.find('removed snapshot') > -1 or result.find('deleted') > -1:
@@ -263,9 +259,6 @@ class IncJobs(multi.Thread):
logging.statusWriter(self.statusPath, 'Failed: %s. [5009]' % (result), 1)
return 0
command = 'restic -r %s prune --password-file %s' % (repoLocation, self.passwordFile)
ProcessUtilities.outputExecutioner(command, self.externalApp)
return 1
else:
repoLocation = '/home/%s/incbackup' % (self.website)
@@ -283,17 +276,17 @@ class IncJobs(multi.Thread):
## Last argument delete is set when the snapshot is to be deleted from this repo, when this argument is set, any preceding argument is not used
def sftpFunction(self, backupPath, type, restore=None, delete=None):
return 0
if restore == None:
# Define our excludes file for use with restic
backupExcludesFile = '/home/%s/backup-exclude.conf' % (self.website.domain)
resticBackupExcludeCMD = ' --exclude-file=%s' % (backupExcludesFile)
remotePath = '/home/backup/%s' % (self.website.domain)
command = 'export PATH=${PATH}:/usr/bin && restic -r %s:%s backup %s --password-file %s --exclude %s --exclude /home/%s/backup' % (
remotePath = '%s' % (self.website.domain)
command = 'export PATH=${PATH}:/usr/bin && restic -r rclone:%s:%s backup %s --password-file %s --exclude %s --exclude /home/%s/backup' % (
self.backupDestinations, remotePath, backupPath, self.passwordFile, self.repoPath, self.website.domain)
# If /home/%s/backup-exclude.conf file exists lets pass this to restic by appending the command to end.
if os.path.isfile(backupExcludesFile):
command = command + resticBackupExcludeCMD
command = command + resticBackupExcludeCMD
result = ProcessUtilities.outputExecutioner(command, self.externalApp)
if result.find('saved') == -1:
@@ -313,9 +306,9 @@ class IncJobs(multi.Thread):
newSnapshot.save()
return 1
elif delete:
repoLocation = '/home/backup/%s' % (self.website)
command = 'export PATH=${PATH}:/usr/bin && restic -r %s:%s forget %s --password-file %s' % (
self.jobid.destination, repoLocation, self.jobid.snapshotid, self.passwordFile)
repoLocation = '%s' % (self.website)
command = 'export PATH=${PATH}:/usr/bin && restic -r rclone:%s:%s forget %s --prune --password-file %s' % (
self.jobid.destination, repoLocation, self.jobids, self.passwordFile)
result = ProcessUtilities.outputExecutioner(command, self.externalApp)
if result.find('removed snapshot') > -1 or result.find('deleted') > -1:
@@ -323,13 +316,10 @@ class IncJobs(multi.Thread):
else:
logging.statusWriter(self.statusPath, 'Failed: %s. [5009]' % (result), 1)
return 0
command = 'export PATH=${PATH}:/usr/bin && restic -r %s:%s prune --password-file %s' % (self.jobid.destination, repoLocation, self.passwordFile)
ProcessUtilities.outputExecutioner(command, self.externalApp)
else:
if self.reconstruct == 'remote':
repoLocation = '/home/backup/%s' % (self.website)
command = 'export RESTIC_PASSWORD=%s PATH=${PATH}:/usr/bin && restic -r %s:%s restore %s --target %s' % (
repoLocation = '%s' % (self.website)
command = 'export RESTIC_PASSWORD=%s PATH=${PATH}:/usr/bin && restic -r rclone:%s:%s restore %s --target %s' % (
self.passwordFile,
self.backupDestinations, repoLocation, self.jobid, self.restoreTarget)
result = ProcessUtilities.outputExecutioner(command, self.externalApp)
@@ -337,8 +327,8 @@ class IncJobs(multi.Thread):
logging.statusWriter(self.statusPath, 'Failed: %s. [5009]' % (result), 1)
return 0
else:
repoLocation = '/home/backup/%s' % (self.website)
command = 'export PATH=${PATH}:/usr/bin && restic -r %s:%s restore %s --target %s --password-file %s' % (
repoLocation = '%s' % (self.website)
command = 'export PATH=${PATH}:/usr/bin && restic -r rclone:%s:%s restore %s --target %s --password-file %s' % (
self.jobid.destination, repoLocation, self.jobid.snapshotid, self.restoreTarget, self.passwordFile)
result = ProcessUtilities.outputExecutioner(command, self.externalApp)
if result.find('restoring') == -1:
@@ -351,7 +341,7 @@ class IncJobs(multi.Thread):
try:
if self.reconstruct == 'remote':
if self.backupDestinations[:4] == 'sftp':
if self.backupDestinations == 'onedrive':
self.sftpFunction('none', 'none', 1)
else:
if self.awsFunction('restore', '', self.jobid) == 0:
@@ -359,7 +349,7 @@ class IncJobs(multi.Thread):
else:
if self.jobid.destination == 'local':
return self.localFunction('none', 'none', 1)
elif self.jobid.destination[:4] == 'sftp':
elif self.jobid.destination == 'onedrive':
return self.sftpFunction('none', 'none', 1)
else:
return self.awsFunction('restore', '', self.jobid.snapshotid)
@@ -374,7 +364,7 @@ class IncJobs(multi.Thread):
try:
if self.reconstruct == 'remote':
if self.backupDestinations[:4] == 'sftp':
if self.backupDestinations == 'onedrive':
if self.sftpFunction('none', 'none', 1) == 0:
return 0
else:
@@ -399,7 +389,7 @@ class IncJobs(multi.Thread):
if self.jobid.destination == 'local':
if self.localFunction('none', 'none', 1) == 0:
return 0
elif self.jobid.destination[:4] == 'sftp':
elif self.jobid.destination == 'onedrive':
if self.sftpFunction('none', 'none', 1) == 0:
return 0
else:
@@ -430,7 +420,7 @@ class IncJobs(multi.Thread):
try:
if self.reconstruct == 'remote':
if self.backupDestinations[:4] == 'sftp':
if self.backupDestinations == 'onedrive':
if self.sftpFunction('none', 'none', 1) == 0:
return 0
else:
@@ -439,7 +429,7 @@ class IncJobs(multi.Thread):
else:
if self.jobid.destination == 'local':
return self.localFunction('none', 'none', 1)
elif self.jobid.destination[:4] == 'sftp':
elif self.jobid.destination == 'onedrive':
return self.sftpFunction('none', 'none', 1)
else:
return self.awsFunction('restore', '', self.jobid.snapshotid)
@@ -454,7 +444,7 @@ class IncJobs(multi.Thread):
try:
if self.reconstruct == 'remote':
if self.backupDestinations[:4] == 'sftp':
if self.backupDestinations == 'onedrive':
if self.sftpFunction('none', 'none', 1) == 0:
return 0
else:
@@ -464,7 +454,7 @@ class IncJobs(multi.Thread):
if self.jobid.destination == 'local':
if self.localFunction('none', 'none', 1) == 0:
return 0
elif self.jobid.destination[:4] == 'sftp':
elif self.jobid.destination == 'onedrive':
if self.sftpFunction('none', 'none', 1) == 0:
return 0
else:
@@ -619,7 +609,7 @@ class IncJobs(multi.Thread):
if self.backupDestinations == 'local':
if self.localFunction(backupPath, 'data') == 0:
return 0
elif self.backupDestinations[:4] == 'sftp':
elif self.backupDestinations == 'onedrive':
if self.sftpFunction(backupPath, 'data') == 0:
return 0
else:
@@ -669,7 +659,7 @@ class IncJobs(multi.Thread):
if self.backupDestinations == 'local':
if self.localFunction(dbPathNew, 'database') == 0:
return 0
elif self.backupDestinations[:4] == 'sftp':
elif self.backupDestinations == 'onedrive':
if self.sftpFunction(dbPathNew, 'database') == 0:
return 0
else:
@@ -700,7 +690,7 @@ class IncJobs(multi.Thread):
if self.backupDestinations == 'local':
if self.localFunction(backupPath, 'email') == 0:
return 0
elif self.backupDestinations[:4] == 'sftp':
elif self.backupDestinations == 'onedrive':
if self.sftpFunction(backupPath, 'email') == 0:
return 0
else:
@@ -723,7 +713,7 @@ class IncJobs(multi.Thread):
if self.backupDestinations == 'local':
if self.localFunction(backupPath, 'meta') == 0:
return 0
elif self.backupDestinations[:4] == 'sftp':
elif self.backupDestinations == 'onedrive':
if self.sftpFunction(backupPath, 'meta') == 0:
return 0
else:
@@ -738,44 +728,42 @@ class IncJobs(multi.Thread):
return 0
def initiateRepo(self):
try:
logging.statusWriter(self.statusPath, 'Will first initiate backup repo..', 1)
try:
logging.statusWriter(self.statusPath, 'Checking for backup repo..', 1)
if self.backupDestinations == 'local':
command = 'restic init --repo %s --password-file %s' % (self.repoPath, self.passwordFile)
result = ProcessUtilities.outputExecutioner(command, self.externalApp)
if os.path.exists(ProcessUtilities.debugPath):
logging.writeToFile(result)
if result.find('config file already exists') == -1:
logging.statusWriter(self.statusPath, result, 1)
elif self.backupDestinations[:4] == 'sftp':
remotePath = '/home/backup/%s' % (self.website.domain)
command = 'export PATH=${PATH}:/usr/bin && restic init --repo %s:%s --password-file %s' % (
self.backupDestinations, remotePath, self.passwordFile)
result = ProcessUtilities.outputExecutioner(command, self.externalApp)
if os.path.exists(ProcessUtilities.debugPath):
logging.writeToFile(result)
if result.find('config file already exists') == -1:
logging.statusWriter(self.statusPath, result, 1)
checkcommand = 'restic --repo %s snapshots --password-file %s' % (self.repoPath, self.passwordFile)
checkresult = ProcessUtilities.outputExecutioner(checkcommand)
if checkresult.find('Is there a repository at the following location') > -1:
logging.statusWriter(self.statusPath, 'Initiating backup repo..', 1)
command = 'restic init --repo %s --password-file %s' % (self.repoPath, self.passwordFile)
elif self.backupDestinations == 'onedrive':
checkcommand = 'restic -r rclone:%s:%s snapshots --password-file %s ' % (self.backupDestinations, self.website.domain, self.passwordFile)
checkresult = ProcessUtilities.outputExecutioner(checkcommand)
if checkresult.find('Is there a repository at the following location') > -1:
logging.statusWriter(self.statusPath, 'Initiating backup repo..', 1)
command = 'export PATH=${PATH}:/usr/bin && restic init --repo rclone:%s:%s --password-file %s' % (
self.backupDestinations, remotePath, self.passwordFile)
else:
key, secret = self.getAWSData()
command = 'export AWS_ACCESS_KEY_ID=%s AWS_SECRET_ACCESS_KEY=%s && restic -r s3:s3.amazonaws.com/%s init --password-file %s' % (
key, secret, self.website.domain, self.passwordFile)
result = ProcessUtilities.outputExecutioner(command, self.externalApp)
checkcommand = 'export AWS_ACCESS_KEY_ID=%s AWS_SECRET_ACCESS_KEY=%s && restic -r s3:s3.amazonaws.com/%s snapshots --password-file %s' % (
key, secret, self.website.domain, self.passwordFile)
checkresult = ProcessUtilities.outputExecutioner(checkcommand, self.externalApp)
if checkresult.find('Is there a repository at the following location') > -1:
logging.statusWriter(self.statusPath, 'Initiating backup repo..', 1)
key, secret = self.getAWSData()
command = 'export AWS_ACCESS_KEY_ID=%s AWS_SECRET_ACCESS_KEY=%s && restic -r s3:s3.amazonaws.com/%s init --password-file %s' % (
key, secret, self.website.domain, self.passwordFile)
if checkresult.find('Is there a repository at the following location') > -1:
result = ProcessUtilities.outputExecutioner(command, self.externalApp)
if os.path.exists(ProcessUtilities.debugPath):
logging.writeToFile(result)
if result.find('config file already exists') == -1:
logging.statusWriter(self.statusPath, result, 1)
logging.statusWriter(self.statusPath,
'Repo %s initiated for %s.' % (self.backupDestinations, self.website.domain), 1)
logging.statusWriter(self.statusPath, 'Repo %s initiated for %s.' % (self.backupDestinations, self.website.domain), 1)
else:
logging.statusWriter(self.statusPath, 'OK', 1)
return 1
except BaseException as msg:
logging.statusWriter(self.statusPath, '%s. [IncJobs.initiateRepo.47][5009]' % str(msg), 1)
@@ -811,9 +799,9 @@ Subject: %s
### Checking if restic is installed before moving on
command = 'restic'
command = 'restic version'
if ProcessUtilities.outputExecutioner(command).find('restic is a backup program which') == -1:
if ProcessUtilities.outputExecutioner(command).find('compiled with') == -1:
try:
CentOSPath = '/etc/redhat-release'
@@ -870,7 +858,7 @@ Subject: %s
## Completed password generation
if self.initiateRepo() == 0:
return 0
return 0
if self.prepareBackupMeta() == 0:
return 0
@@ -921,24 +909,21 @@ Subject: %s
self.website = job_snapshots[0].job.website.domain
self.externalApp = job_snapshots[0].job.website.externalApp
self.passwordFile = '/home/%s/%s' % (self.website, self.website)
for job_snapshot in job_snapshots:
## Functions above use the self.jobid varilable to extract information about this snapshot, so this below variable needs to be set
## Functions above use the self.jobid variable to extract information about this snapshot, so this below variable needs to be set
self.jobid = job_snapshot
self.jobids = self.jobids + job_snapshot.snapshotid + ' '
if self.jobid.destination == 'local':
self.localFunction('none', 'none', 0, 1)
elif self.jobid.destination[:4] == 'sftp':
self.sftpFunction('none', 'none', 0, 1)
else:
self.awsFunction('restore', '', self.jobid.snapshotid, None, 1)
if self.jobid.destination == 'local':
self.localFunction('none', 'none', 0, 1)
elif self.jobid.destination == 'onedrive':
self.sftpFunction('none', 'none', 0, 1)
else:
self.awsFunction('restore', '', 0, None, 1)
return 1
except BaseException as msg:
logging.statusWriter(self.statusPath, "%s [903:DeleteSnapShot][5009]" % (str(msg)), 1)
return 0

View File

@@ -35,7 +35,7 @@
<label class="col-sm-3 control-label">{% trans "Select Type" %} </label>
<div class="col-sm-6">
<select ng-change="fetchDetails()" ng-model="destinationType" class="form-control">
{# <option>SFTP</option>#}
<option>SFTP</option>
<option>AWS</option>
</select>
</div>
@@ -181,4 +181,4 @@
</div>
{% endblock %}
{% endblock %}

View File

@@ -291,7 +291,7 @@ def submit_backup_creation(request):
data = json.loads(request.body)
backup_domain = data['websiteToBeBacked']
backup_destinations = data['backupDestinations']
backup_destinations = 'onedrive'
if ACLManager.checkOwnership(backup_domain, admin, current_acl) == 1:
pass

View File

@@ -145,22 +145,22 @@ if __name__ == "__main__":
writeToFile.write(str(os.getpid()))
writeToFile.close()
SSHServer.findSSHPort()
http_server = tornado.httpserver.HTTPServer(application, ssl_options={
"certfile": "/usr/local/lscp/conf/cert.pem",
"keyfile": "/usr/local/lscp/conf/key.pem",
}, )
ADDR = '0.0.0.0'
http_server.listen(5678, ADDR)
print('*** Websocket Server Started at %s***' % ADDR)
import signal
def close_sig_handler(signal, frame):
http_server.stop()
sys.exit()
signal.signal(signal.SIGINT, close_sig_handler)
tornado.ioloop.IOLoop.instance().start()
# SSHServer.findSSHPort()
#
# http_server = tornado.httpserver.HTTPServer(application, ssl_options={
# "certfile": "/usr/local/lscp/conf/cert.pem",
# "keyfile": "/usr/local/lscp/conf/key.pem",
# }, )
#
# ADDR = '0.0.0.0'
# http_server.listen(5678, ADDR)
# print('*** Websocket Server Started at %s***' % ADDR)
#
# import signal
# def close_sig_handler(signal, frame):
# http_server.stop()
# sys.exit()
#
# signal.signal(signal.SIGINT, close_sig_handler)
#
# tornado.ioloop.IOLoop.instance().start()

View File

@@ -699,4 +699,6 @@ app.controller('designtheme', function ($scope, $http, $timeout) {
//$timeout(getStuff, 2000);
};
});
});

View File

@@ -37,6 +37,7 @@ def getAdminStatus(request):
val = request.session['userID']
currentACL = ACLManager.loadedACL(val)
if os.path.exists('/home/cyberpanel/postfix'):
currentACL['emailAsWhole'] = 1
else:

View File

@@ -141,8 +141,8 @@ class EmailMarketingManager:
if emACL.checkIfEMEnabled(admin.userName) == 0:
return ACLManager.loadErrorJson()
em = EM('createEmailList', extraArgs)
em.start()
# em = EM('createEmailList', extraArgs)
# em.start()
time.sleep(2)

View File

@@ -546,9 +546,11 @@ class FileManager:
finalData = {}
finalData['status'] = 1
tempPath = "/home/cyberpanel/" + str(randint(1000, 9999))
self.data['home'] = '/home/%s' % (self.data['domainName'])
ACLManager.CreateSecureDir()
tempPath = '%s/%s' % ('/usr/local/CyberCP/tmp', str(randint(1000, 9999)))
domainName = self.data['domainName']
website = Websites.objects.get(domain=domainName)
@@ -556,31 +558,13 @@ class FileManager:
writeToFile.write(self.data['fileContent'].encode('utf-8'))
writeToFile.close()
command = 'ls -la %s' % (self.data['fileName'])
output = ProcessUtilities.outputExecutioner(command)
if output.find('lrwxrwxrwx') > -1 and output.find('->') > -1:
return self.ajaxPre(0, 'File exists and is symlink.')
if ACLManager.commandInjectionCheck(self.data['fileName']) == 1:
return self.ajaxPre(0, 'Not allowed to move in this path, please choose location inside home!')
if self.data['fileName'].find(self.data['home']) == -1 or self.data['fileName'].find('..') > -1:
return self.ajaxPre(0, 'Not allowed to move in this path, please choose location inside home!')
command = 'stat -c "%%a" %s' % (self.returnPathEnclosed(self.data['fileName']))
currentMode = ProcessUtilities.outputExecutioner(command).strip('\n')
command = 'mv ' + tempPath + ' ' + self.returnPathEnclosed(self.data['fileName'])
command = 'chown %s:%s %s' % (website.externalApp, website.externalApp, tempPath)
ProcessUtilities.executioner(command)
command = 'chown %s:%s %s' % (website.externalApp, website.externalApp, self.data['fileName'])
ProcessUtilities.executioner(command)
command = 'cp %s %s' % (tempPath, self.returnPathEnclosed(self.data['fileName']))
ProcessUtilities.executioner(command, website.externalApp)
command = 'chmod %s %s' % (currentMode, self.returnPathEnclosed(self.data['fileName']))
ProcessUtilities.executioner(command)
self.changeOwner(self.data['fileName'])
os.remove(tempPath)
json_data = json.dumps(finalData)
return HttpResponse(json_data)

View File

@@ -2660,7 +2660,7 @@ vmail
command = 'csf -uf'
Upgrade.executioner(command, 'fix csf if there', 0)
command = 'systemctl start cpssh'
command = 'systemctl stop cpssh'
Upgrade.executioner(command, 'fix csf if there', 0)
Upgrade.AutoUpgradeAcme()
Upgrade.installCLScripts()

View File

@@ -653,4 +653,5 @@ app.controller('versionManagment', function ($scope, $http, $timeout) {
};
});
});