diff --git a/plogical/IncScheduler.py b/plogical/IncScheduler.py index 46dc66c1a..a130fafb2 100644 --- a/plogical/IncScheduler.py +++ b/plogical/IncScheduler.py @@ -476,6 +476,28 @@ Automatic backup failed for %s on %s. subprocess.call(shlex.split(command)) if jobConfig[IncScheduler.frequency] == type: + + ### Check if an old job prematurely killed, then start from there. + try: + oldJobContinue = 1 + pid = jobConfig['pid'] + stuckDomain = jobConfig['website'] + finalPath = jobConfig['finalPath'] + jobConfig['pid'] = os.getpid() + + command = 'ps aux' + result = ProcessUtilities.outputExecutioner(command) + + if result.find(pid) > -1 and result.find('IncScheduler.py') > -1: + quit(1) + + + except: + ### Save some important info in backup config + oldJobContinue = 0 + jobConfig['pid'] = str(os.getpid()) + jobConfig['finalPath'] = finalPath + NormalBackupJobLogs.objects.filter(owner=backupjob).delete() NormalBackupJobLogs(owner=backupjob, status=backupSchedule.INFO, message='Starting %s backup on %s..' % ( @@ -491,6 +513,8 @@ Automatic backup failed for %s on %s. except: websites = backupjob.normalbackupsites_set.all().order_by('domain__domain') + doit = 0 + for site in websites: if actualDomain: @@ -498,6 +522,19 @@ Automatic backup failed for %s on %s. else: domain = site.domain.domain + ## Save currently backing domain in db, so that i can restart from here when prematurely killed + + jobConfig['website'] = domain + backupjob.config = json.dumps(jobConfig) + backupjob.save() + + if oldJobContinue and not doit: + if domain == stuckDomain: + doit = 1 + pass + else: + continue + retValues = backupSchedule.createLocalBackup(domain, '/dev/null') if retValues[0] == 0: @@ -529,6 +566,12 @@ Automatic backup failed for %s on %s. message='Backup completed for %s on %s.' % ( domain, time.strftime("%m.%d.%Y_%H-%M-%S"))).save() + jobConfig = json.loads(backupjob.config) + if jobConfig['pid']: + del jobConfig['pid'] + backupjob.config = json.dumps(jobConfig) + backupjob.save() + def main():