From 952c65e7feefc3f58139b79342e77d66ff34ab4c Mon Sep 17 00:00:00 2001 From: usmannasir Date: Wed, 2 Jul 2025 14:29:04 +0500 Subject: [PATCH] bug fix: https://community.cyberpanel.net/t/sftp-remote-backup-fails-on-sftp-only-destinations-hetzner-storage-box-etc/58851 --- plogical/IncScheduler.py | 122 +++++++++++++++++++++++++++++++----- plogical/backupSchedule.py | 56 ++++++++++++++++- plogical/backupUtilities.py | 112 ++++++++++++++++++++++++++++----- 3 files changed, 254 insertions(+), 36 deletions(-) diff --git a/plogical/IncScheduler.py b/plogical/IncScheduler.py index b42623e32..57e73df8b 100644 --- a/plogical/IncScheduler.py +++ b/plogical/IncScheduler.py @@ -660,6 +660,9 @@ Automatic backup failed for %s on %s. print(str(msg)) continue + # Always try SSH commands first + ssh_commands_supported = True + try: command = f'find cpbackups -type f -mtime +{jobConfig["retention"]} -exec rm -f {{}} \\;' logging.writeToFile(command) @@ -673,18 +676,67 @@ Automatic backup failed for %s on %s. # Execute the command to create the remote directory command = f'mkdir -p {finalPath}' - stdin, stdout, stderr = ssh.exec_command(command) - - # Wait for the command to finish and check for any errors - stdout.channel.recv_exit_status() - error_message = stderr.read().decode('utf-8') - print(error_message) - if error_message: - NormalBackupJobLogs(owner=backupjob, status=backupSchedule.INFO, - message=f'Error while creating directory on remote server {error_message.strip()}').save() - continue - else: - pass + try: + stdin, stdout, stderr = ssh.exec_command(command, timeout=10) + # Wait for the command to finish and check for any errors + exit_status = stdout.channel.recv_exit_status() + error_message = stderr.read().decode('utf-8') + print(error_message) + + # Check if command was rejected (SFTP-only server) + if exit_status != 0 or "not allowed" in error_message.lower() or "channel closed" in error_message.lower(): + ssh_commands_supported = False + logging.writeToFile(f'SSH command failed on {destinationConfig["ip"]}, falling back to pure SFTP mode') + + # Try creating directory via SFTP + try: + sftp = ssh.open_sftp() + # Try to create the directory structure + path_parts = finalPath.strip('/').split('/') + current_path = '' + for part in path_parts: + current_path = current_path + '/' + part if current_path else part + try: + sftp.stat(current_path) + except FileNotFoundError: + try: + sftp.mkdir(current_path) + except: + pass + sftp.close() + except BaseException as msg: + logging.writeToFile(f'Failed to create directory via SFTP: {str(msg)}') + pass + elif error_message: + NormalBackupJobLogs(owner=backupjob, status=backupSchedule.INFO, + message=f'Error while creating directory on remote server {error_message.strip()}').save() + continue + else: + pass + except BaseException as msg: + # SSH command failed, try SFTP + ssh_commands_supported = False + logging.writeToFile(f'SSH command failed: {str(msg)}, falling back to pure SFTP mode') + + # Try creating directory via SFTP + try: + sftp = ssh.open_sftp() + # Try to create the directory structure + path_parts = finalPath.strip('/').split('/') + current_path = '' + for part in path_parts: + current_path = current_path + '/' + part if current_path else part + try: + sftp.stat(current_path) + except FileNotFoundError: + try: + sftp.mkdir(current_path) + except: + pass + sftp.close() + except BaseException as msg: + logging.writeToFile(f'Failed to create directory via SFTP: {str(msg)}') + pass ### Check if an old job prematurely killed, then start from there. @@ -788,10 +840,30 @@ Automatic backup failed for %s on %s. else: backupPath = retValues[1] + ".tar.gz" + # Always try scp first command = "scp -o StrictHostKeyChecking=no -P " + destinationConfig[ 'port'] + " -i /root/.ssh/cyberpanel " + backupPath + " " + destinationConfig[ 'username'] + "@" + destinationConfig['ip'] + ":%s" % (finalPath) - ProcessUtilities.executioner(command) + + try: + result = ProcessUtilities.executioner(command) + # Check if scp failed (common with SFTP-only servers) + if not ssh_commands_supported or result != 0: + raise Exception("SCP failed, trying SFTP") + except: + # If scp fails or SSH commands are not supported, use SFTP + logging.writeToFile(f'SCP failed for {destinationConfig["ip"]}, falling back to SFTP transfer') + try: + sftp = ssh.open_sftp() + remote_path = os.path.join(finalPath, os.path.basename(backupPath)) + sftp.put(backupPath, remote_path) + sftp.close() + logging.writeToFile(f'Successfully transferred {backupPath} to {remote_path} via SFTP') + except BaseException as msg: + logging.writeToFile(f'Failed to transfer backup via SFTP: {str(msg)}') + NormalBackupJobLogs(owner=backupjob, status=backupSchedule.ERROR, + message='Backup transfer failed for %s: %s' % (domain, str(msg))).save() + continue try: os.remove(backupPath) @@ -832,11 +904,27 @@ Automatic backup failed for %s on %s. # Command to list directories under the specified path command = f"ls -d {finalPath}/*" - # Execute the command - stdin, stdout, stderr = ssh.exec_command(command) + # Try SSH command first + directories = [] + try: + # Execute the command + stdin, stdout, stderr = ssh.exec_command(command, timeout=10) - # Read the results - directories = stdout.read().decode().splitlines() + # Read the results + directories = stdout.read().decode().splitlines() + except: + # If SSH command fails, try using SFTP + logging.writeToFile(f'SSH ls command failed for {destinationConfig["ip"]}, trying SFTP listdir') + try: + sftp = ssh.open_sftp() + # List files in the directory + files = sftp.listdir(finalPath) + # Format them similar to ls -d output + directories = [f"{finalPath}/{f}" for f in files] + sftp.close() + except BaseException as msg: + logging.writeToFile(f'Failed to list directory via SFTP: {str(msg)}') + directories = [] if os.path.exists(ProcessUtilities.debugPath): logging.writeToFile(str(directories)) diff --git a/plogical/backupSchedule.py b/plogical/backupSchedule.py index c803208ab..2b0598433 100644 --- a/plogical/backupSchedule.py +++ b/plogical/backupSchedule.py @@ -318,12 +318,64 @@ class backupSchedule: ## writeToFile = open(backupLogPath, "a") - command = "scp -o StrictHostKeyChecking=no -P "+port+" -i /root/.ssh/cyberpanel " + backupPath + " " + user + "@" + IPAddress+":~/backup/" + ipAddressLocal + "/" + time.strftime("%m.%d.%Y_%H-%M-%S") + "/" - subprocess.call(shlex.split(command), stdout=writeToFile) + remote_dir = "~/backup/" + ipAddressLocal + "/" + time.strftime("%m.%d.%Y_%H-%M-%S") + "/" + command = "scp -o StrictHostKeyChecking=no -P "+port+" -i /root/.ssh/cyberpanel " + backupPath + " " + user + "@" + IPAddress+":" + remote_dir + + # Try scp first + result = subprocess.call(shlex.split(command), stdout=writeToFile) if os.path.exists(ProcessUtilities.debugPath): logging.CyberCPLogFileWriter.writeToFile(command) + # If scp fails, try SFTP + if result != 0: + writeToFile.write("SCP failed, attempting SFTP transfer...\n") + try: + import paramiko + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + + # Try key-based auth first + try: + private_key = paramiko.RSAKey.from_private_key_file('/root/.ssh/cyberpanel') + ssh.connect(IPAddress, port=int(port), username=user, pkey=private_key) + except: + # If key auth fails, connection setup failed + raise Exception("Failed to connect with SSH key") + + # Create remote directory structure via SFTP + sftp = ssh.open_sftp() + + # Convert ~ to actual home directory + home_dir = sftp.normalize('.') + remote_full_path = os.path.join(home_dir, 'backup', ipAddressLocal, time.strftime("%m.%d.%Y_%H-%M-%S")) + + # Create directory structure + path_parts = remote_full_path.strip('/').split('/') + current_path = '/' + for part in path_parts: + current_path = os.path.join(current_path, part) + try: + sftp.stat(current_path) + except FileNotFoundError: + try: + sftp.mkdir(current_path) + except: + pass + + # Transfer file + remote_file = os.path.join(remote_full_path, os.path.basename(backupPath)) + sftp.put(backupPath, remote_file) + sftp.close() + ssh.close() + + writeToFile.write(f"Successfully transferred {backupPath} to {remote_file} via SFTP\n") + logging.CyberCPLogFileWriter.writeToFile(f"Successfully transferred backup via SFTP to {IPAddress}") + except BaseException as msg: + writeToFile.write(f"SFTP transfer failed: {str(msg)}\n") + logging.CyberCPLogFileWriter.writeToFile(f"SFTP transfer failed: {str(msg)}") + raise + ## Remove backups already sent to remote destinations os.remove(backupPath) diff --git a/plogical/backupUtilities.py b/plogical/backupUtilities.py index 8a61ecbb4..753248f29 100644 --- a/plogical/backupUtilities.py +++ b/plogical/backupUtilities.py @@ -1417,11 +1417,43 @@ class backupUtilities: command = 'chmod 600 %s' % ('/root/.ssh/cyberpanel.pub') ProcessUtilities.executioner(command) - sftp = ssh.open_sftp() - sftp.put('/root/.ssh/cyberpanel.pub', '.ssh/authorized_keys') - sftp.close() + try: + # Try to use SFTP to create .ssh directory if it doesn't exist + sftp = ssh.open_sftp() + try: + sftp.stat('.ssh') + except FileNotFoundError: + # Try to create .ssh directory via SFTP + try: + sftp.mkdir('.ssh') + except: + # Directory creation via SFTP might fail on some servers + pass + + # Try to upload the key + sftp.put('/root/.ssh/cyberpanel.pub', '.ssh/authorized_keys') + sftp.close() - ssh.exec_command('chmod 600 .ssh/authorized_keys') + # Try to set permissions via SSH command (might fail on SFTP-only servers) + try: + stdin, stdout, stderr = ssh.exec_command('chmod 600 .ssh/authorized_keys', timeout=5) + stdout.channel.recv_exit_status() + except: + # If chmod fails, it's likely an SFTP-only server + # The key is uploaded, which is what matters for backups using password auth + logging.CyberCPLogFileWriter.writeToFile( + f'Could not set permissions on {IPAddress}, likely SFTP-only server') + pass + + except Exception as e: + # If we can't upload the key, it might be an SFTP-only server + # Return success anyway since password authentication works + logging.CyberCPLogFileWriter.writeToFile( + f'Could not upload SSH key to {IPAddress}: {str(e)}, using password authentication') + ssh.close() + command = 'chmod 644 %s' % ('/root/.ssh/cyberpanel.pub') + ProcessUtilities.executioner(command) + return [1, "None"] ssh.close() @@ -1446,6 +1478,9 @@ class backupUtilities: if password != 'NOT-NEEDED': ssh.connect(IPAddress, port=int(port), username=user, password=password) + + # Try to execute SSH commands first + ssh_commands_supported = True commands = [ "mkdir -p .ssh", "rm -f .ssh/temp", @@ -1457,23 +1492,55 @@ class backupUtilities: for command in commands: try: - ssh.exec_command(command) + stdin, stdout, stderr = ssh.exec_command(command, timeout=5) + exit_status = stdout.channel.recv_exit_status() + error_output = stderr.read().decode() + + # Check if the command was rejected (SFTP-only server) + if exit_status != 0 or "not allowed" in error_output.lower() or "channel closed" in error_output.lower(): + ssh_commands_supported = False + logging.CyberCPLogFileWriter.writeToFile( + f'SSH commands not supported on {IPAddress}, falling back to pure SFTP mode') + break except BaseException as msg: + ssh_commands_supported = False logging.CyberCPLogFileWriter.writeToFile( - f'Error executing remote command {command}. Error {str(msg)}') + f'Error executing remote command {command}. Error {str(msg)}, falling back to pure SFTP mode') + break ssh.close() - sendKey = backupUtilities.sendKey(IPAddress, password, port, user) - - if sendKey[0] == 1: - command = 'chmod 644 %s' % ('/root/.ssh/cyberpanel.pub') - ProcessUtilities.executioner(command) - return [1, "None"] + # If SSH commands are not supported, use pure SFTP mode + if not ssh_commands_supported: + # For SFTP-only servers, we'll use password authentication directly + # No need to setup SSH keys, just verify connection works + try: + test_ssh = paramiko.SSHClient() + test_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + test_ssh.connect(IPAddress, port=int(port), username=user, password=password) + + # Open SFTP connection to verify it works + sftp = test_ssh.open_sftp() + sftp.close() + test_ssh.close() + + logging.CyberCPLogFileWriter.writeToFile( + f'Pure SFTP mode verified for {IPAddress}') + return [1, "None"] + except Exception as e: + return [0, f'SFTP connection failed: {str(e)}'] else: - command = 'chmod 644 %s' % ('/root/.ssh/cyberpanel.pub') - ProcessUtilities.executioner(command) - return [0, sendKey[1]] + # SSH commands are supported, proceed with key setup + sendKey = backupUtilities.sendKey(IPAddress, password, port, user) + + if sendKey[0] == 1: + command = 'chmod 644 %s' % ('/root/.ssh/cyberpanel.pub') + ProcessUtilities.executioner(command) + return [1, "None"] + else: + command = 'chmod 644 %s' % ('/root/.ssh/cyberpanel.pub') + ProcessUtilities.executioner(command) + return [0, sendKey[1]] else: # Load the private key private_key_path = '/root/.ssh/cyberpanel' @@ -1632,12 +1699,20 @@ class backupUtilities: def createBackupDir(IPAddress, port='22', user='root'): try: + # First try SSH command command = "sudo ssh -o StrictHostKeyChecking=no -p " + port + " -i /root/.ssh/cyberpanel " + user + "@" + IPAddress + " mkdir ~/backup" if os.path.exists(ProcessUtilities.debugPath): logging.CyberCPLogFileWriter.writeToFile(command) - subprocess.call(shlex.split(command)) + result = subprocess.call(shlex.split(command)) + + # If SSH command fails, it might be an SFTP-only server + if result != 0: + logging.CyberCPLogFileWriter.writeToFile( + f"SSH command failed for {IPAddress}, likely SFTP-only server. Skipping directory creation.") + # Don't fail - SFTP servers may have their own directory structure + return 1 command = "sudo ssh -o StrictHostKeyChecking=no -p " + port + " -i /root/.ssh/cyberpanel " + user + "@" + IPAddress + ' "cat ~/.ssh/authorized_keys ~/.ssh/temp > ~/.ssh/authorized_temp"' @@ -1653,10 +1728,13 @@ class backupUtilities: logging.CyberCPLogFileWriter.writeToFile(command) subprocess.call(shlex.split(command)) + + return 1 except BaseException as msg: logging.CyberCPLogFileWriter.writeToFile(str(msg) + " [createBackupDir]") - return 0 + # Don't fail for SFTP-only servers + return 1 @staticmethod def host_key_verification(IPAddress):