From 53fc6a52e50372b492bdf35cbc1f9150687ac439 Mon Sep 17 00:00:00 2001 From: master3395 Date: Fri, 6 Mar 2026 18:50:20 +0100 Subject: [PATCH] Docker containers 500 fix, firewall banned IPs, container logs readability, base Ban IP sync to DB - dockerManager: add 0001_initial migration (CREATE TABLE IF NOT EXISTS), migrate-and-retry on DB errors, safe error response, fix logging.CyberCPLogFileWriter.writeToFile - dockerManager/views: listContainersPage fallback HTML with error message if template fails - dockerManager/viewContainer: improve container log readability (font-size 1rem, color #f1f5f9, line-height 1.6) - baseTemplate: blockIPAddress also adds ban to firewall BannedIP model so Firewall > Banned IPs shows all - firewall: getBannedIPs migrate-and-retry on OperationalError/ProgrammingError; install runs migrate firewall - plogical/upgrade: syncBannedIPsJsonToDb() to sync JSON bans to firewall_bannedips; firewallMigrations() calls it; CyberPanelUpgrade runs firewallMigrations(); someDirectories creates /usr/local/CyberCP/data - install: explicit migrate firewall after global migrate --- baseTemplate/views.py | 21 +++ dockerManager/container.py | 61 +++++++-- dockerManager/migrations/0001_initial.py | 53 ++++++++ .../dockerManager/viewContainer.html | 9 +- dockerManager/views.py | 31 ++++- firewall/firewallManager.py | 74 ++++++---- install/install.py | 5 + plogical/CyberPanelUpgrade.py | 1 + plogical/upgrade.py | 127 +++++++++++++++++- 9 files changed, 325 insertions(+), 57 deletions(-) create mode 100644 dockerManager/migrations/0001_initial.py diff --git a/baseTemplate/views.py b/baseTemplate/views.py index a67ddcf74..510311506 100644 --- a/baseTemplate/views.py +++ b/baseTemplate/views.py @@ -1454,6 +1454,27 @@ def blockIPAddress(request): # Save to file with open(primary_file, 'w') as f: json.dump(banned_ips, f, indent=2) + + # Also add to firewall DB so it shows on Firewall > Banned IPs + try: + from firewall.models import BannedIP + from django.utils import timezone + user_id = request.session.get('userID') + if user_id: + admin = Administrator.objects.get(pk=user_id) + BannedIP.objects.get_or_create( + ip_address=ip_address, + defaults={ + 'reason': reason, + 'duration': 'permanent', + 'banned_on': timezone.now(), + 'expires': None, + 'active': True, + 'admin': admin, + } + ) + except Exception as db_e: + logging.CyberCPLogFileWriter.writeToFile(f'Warning: Failed to add banned IP to firewall DB: {str(db_e)}') except Exception as e: # Log but don't fail the request if JSON update fails import plogical.CyberCPLogFileWriter as logging diff --git a/dockerManager/container.py b/dockerManager/container.py index 028f2edc2..efdbc3635 100644 --- a/dockerManager/container.py +++ b/dockerManager/container.py @@ -16,7 +16,7 @@ import plogical.CyberCPLogFileWriter as logging from plogical.errorSanitizer import secure_error_response, secure_log_error from django.shortcuts import HttpResponse, render, redirect from django.urls import reverse -from django.db.utils import OperationalError +from django.db.utils import OperationalError, ProgrammingError, InternalError from loginSystem.models import Administrator import subprocess import shlex @@ -258,32 +258,69 @@ class ContainerManager(multi.Thread): "showUnlistedContainer": showUnlistedContainer}, 'admin') return proc.render() - try: - return _render_list() - except OperationalError as e: - logging.writeToFile( - "Docker containers list: DB error (table may be missing). Running migrations. Error: %s" % str(e) + def _run_migrate_and_retry(exc): + logging.CyberCPLogFileWriter.writeToFile( + "Docker containers list: DB error (table/column may be missing). Running migrations. Error: %s" % str(exc) ) try: + # Ensure table exists: raw SQL path (idempotent) then Django migrate + try: + from plogical.upgrade import Upgrade + Upgrade.dockerMigrations() + except Exception as _: + pass from django.core.management import call_command call_command('migrate', 'dockerManager', verbosity=0) return _render_list() except Exception as migrate_err: - logging.writeToFile( + logging.CyberCPLogFileWriter.writeToFile( "Docker containers list: migrate failed. Error: %s" % str(migrate_err) ) + return _safe_error_response( + request, + 'Docker Manager database not ready. Please run upgrade or: manage.py migrate dockerManager', + status=500 + ) + + def _safe_error_response(request, message, status=500): + """Return error page or minimal HttpResponse if template render fails.""" + try: return render( request, 'baseTemplate/error.html', - {'error_message': 'Docker Manager database not ready. Please run upgrade or: manage.py migrate dockerManager'} + {'error_message': message}, + status=status ) + except Exception as render_err: + logging.CyberCPLogFileWriter.writeToFile("Docker listContainers: render error.html failed: %s" % str(render_err)) + safe_msg = (message or "Error")[:500].replace("<", "<").replace(">", ">") + html = "

Docker Containers

%s

" % safe_msg + return HttpResponse(html, status=status) + + try: + return _render_list() + except OperationalError as e: + return _run_migrate_and_retry(e) + except ProgrammingError as e: + return _run_migrate_and_retry(e) + except InternalError as e: + return _run_migrate_and_retry(e) except Exception as e: + import traceback secure_log_error(e, 'docker_list_containers') - return render( - request, - 'baseTemplate/error.html', - {'error_message': 'Containers list could not be loaded. Check error logs.'} + logging.CyberCPLogFileWriter.writeToFile( + "Docker containers list: %s: %s" % (type(e).__name__, str(e)) ) + logging.CyberCPLogFileWriter.writeToFile("Docker containers list traceback:\n%s" % traceback.format_exc()) + # User-friendly message for common Docker errors + err_msg = str(e) + if hasattr(e, 'explicit') and getattr(e, 'explicit', None): + err_msg = getattr(e, 'explicit', err_msg) or err_msg + if 'docker' in type(e).__name__.lower() or 'connection' in err_msg.lower() or 'refused' in err_msg.lower(): + message = 'Docker is not responding. Ensure the Docker daemon is running and try again. Error: %s' % (err_msg[:150]) + else: + message = 'Containers list could not be loaded. Check error logs.' + return _safe_error_response(request, message, status=500) def getContainerLogs(self, userID=None, data=None): try: diff --git a/dockerManager/migrations/0001_initial.py b/dockerManager/migrations/0001_initial.py new file mode 100644 index 000000000..710d3ab9e --- /dev/null +++ b/dockerManager/migrations/0001_initial.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +# Docker Manager initial migration. +# Creates dockerManager_containers table if not exists (safe when upgrade.dockerMigrations() already ran). + +from django.db import migrations + + +def create_table_if_not_exists(apps, schema_editor): + """Create dockerManager_containers with full schema. Idempotent via IF NOT EXISTS.""" + if schema_editor.connection.vendor != 'mysql': + return + # Use raw SQL so we can do IF NOT EXISTS; matches upgrade.dockerMigrations() final schema + sql = """ + CREATE TABLE IF NOT EXISTS dockerManager_containers ( + id int(11) NOT NULL AUTO_INCREMENT, + name varchar(150) NOT NULL, + cid varchar(64) NOT NULL DEFAULT '', + admin_id int(11) NOT NULL, + image varchar(50) NOT NULL DEFAULT 'unknown', + tag varchar(50) NOT NULL DEFAULT 'unknown', + memory int(11) NOT NULL DEFAULT 0, + ports longtext NOT NULL, + volumes longtext NOT NULL, + env longtext NOT NULL, + startOnReboot int(11) NOT NULL DEFAULT 0, + network varchar(100) NOT NULL DEFAULT 'bridge', + network_mode varchar(50) NOT NULL DEFAULT 'bridge', + extra_options longtext NOT NULL, + PRIMARY KEY (id), + UNIQUE KEY name (name), + KEY dockerManager_contai_admin_id_58fb62b7_fk_loginSyst (admin_id), + CONSTRAINT dockerManager_contai_admin_id_58fb62b7_fk_loginSyst + FOREIGN KEY (admin_id) REFERENCES loginSystem_administrator (id) + ) + """ + schema_editor.execute(sql) + + +def noop_reverse(apps, schema_editor): + pass + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('loginSystem', '__first__'), + ] + + operations = [ + migrations.RunPython(create_table_if_not_exists, noop_reverse), + ] diff --git a/dockerManager/templates/dockerManager/viewContainer.html b/dockerManager/templates/dockerManager/viewContainer.html index 3fdca0479..bc9b2087a 100644 --- a/dockerManager/templates/dockerManager/viewContainer.html +++ b/dockerManager/templates/dockerManager/viewContainer.html @@ -429,15 +429,16 @@ } .terminal-content { - font-family: 'SF Mono', Monaco, Consolas, monospace; - font-size: 0.8125rem; - line-height: 1.5; - color: #e2e8f0; + font-family: 'SF Mono', Monaco, Consolas, 'Courier New', monospace; + font-size: 1rem; + line-height: 1.6; + color: #f1f5f9; background: #1a202c; padding: 1.5rem; height: 400px; overflow-y: auto; white-space: pre-wrap; + word-wrap: break-word; } .terminal-content::-webkit-scrollbar { diff --git a/dockerManager/views.py b/dockerManager/views.py index 9bf1b9f0c..bc9896e32 100644 --- a/dockerManager/views.py +++ b/dockerManager/views.py @@ -197,15 +197,32 @@ def listContainersPage(request): except KeyError: return redirect(loadLoginPage) except Exception as e: + import traceback from django.shortcuts import render + from django.http import HttpResponse from plogical.CyberCPLogFileWriter import CyberCPLogFileWriter as logging - logging.writeToFile("listContainersPage error: %s" % str(e)) - return render( - request, - 'baseTemplate/error.html', - {'error_message': 'Containers page could not be loaded. Check error logs.'}, - status=500 - ) + err_msg = str(e) + err_type = type(e).__name__ + logging.writeToFile("listContainersPage error: %s: %s" % (err_type, err_msg)) + logging.writeToFile("listContainersPage traceback:\n%s" % traceback.format_exc()) + # Try standard error template first; if it fails (e.g. missing), return minimal HTML so user sees the error + try: + return render( + request, + 'baseTemplate/error.html', + {'error_message': 'Containers page could not be loaded. Check error logs.'}, + status=500 + ) + except Exception as render_err: + logging.writeToFile("listContainersPage: render error.html failed: %s" % str(render_err)) + safe_msg = err_type + ": " + (err_msg[:200] if err_msg else "Unknown error") + html = ( + "Docker Containers Error" + "

Containers page error

%s

" + "

Check /home/cyberpanel/error-logs.txt for full traceback.

" + "" + ) % (safe_msg.replace("<", "<").replace(">", ">")) + return HttpResponse(html, status=500) @preDockerRun diff --git a/firewall/firewallManager.py b/firewall/firewallManager.py index 6db546e11..ea3d030ab 100644 --- a/firewall/firewallManager.py +++ b/firewall/firewallManager.py @@ -1924,37 +1924,55 @@ class FirewallManager: active_banned_ips = [] db_ips = set() # IPs already added from DB (for merge with JSON) current_time = int(time.time()) + from django.db.utils import OperationalError, ProgrammingError - try: - from firewall.models import BannedIP - from django.db.models import Q + for _migrate_attempt in (1, 2): + try: + from firewall.models import BannedIP + from django.db.models import Q - banned_ips_queryset = BannedIP.objects.filter( - active=True - ).filter( - Q(expires__isnull=True) | Q(expires__gt=current_time) - ).order_by('-banned_on') + banned_ips_queryset = BannedIP.objects.filter( + active=True + ).filter( + Q(expires__isnull=True) | Q(expires__gt=current_time) + ).order_by('-banned_on') - for banned_ip in banned_ips_queryset: - try: - ip_data = { - 'id': banned_ip.id, - 'ip': banned_ip.ip_address, - 'reason': banned_ip.reason, - 'duration': banned_ip.duration, - 'banned_on': banned_ip.get_banned_on_display(), - 'expires': banned_ip.get_expires_display(), - 'active': not banned_ip.is_expired() and banned_ip.active - } - if ip_data['active']: - active_banned_ips.append(ip_data) - db_ips.add(banned_ip.ip_address) - except Exception as row_e: - import plogical.CyberCPLogFileWriter as _log - _log.CyberCPLogFileWriter.writeToFile('getBannedIPs: skip row %s: %s' % (getattr(banned_ip, 'ip_address', '?'), str(row_e))) - except Exception as e: - import plogical.CyberCPLogFileWriter as _log - _log.CyberCPLogFileWriter.writeToFile('getBannedIPs: DB read failed, merging with JSON (%s)' % str(e)) + for banned_ip in banned_ips_queryset: + try: + ip_data = { + 'id': banned_ip.id, + 'ip': banned_ip.ip_address, + 'reason': banned_ip.reason, + 'duration': banned_ip.duration, + 'banned_on': banned_ip.get_banned_on_display(), + 'expires': banned_ip.get_expires_display(), + 'active': not banned_ip.is_expired() and banned_ip.active + } + if ip_data['active']: + active_banned_ips.append(ip_data) + db_ips.add(banned_ip.ip_address) + except Exception as row_e: + import plogical.CyberCPLogFileWriter as _log + _log.CyberCPLogFileWriter.writeToFile('getBannedIPs: skip row %s: %s' % (getattr(banned_ip, 'ip_address', '?'), str(row_e))) + break + except (OperationalError, ProgrammingError) as e: + import plogical.CyberCPLogFileWriter as _log + _log.CyberCPLogFileWriter.writeToFile('getBannedIPs: DB error (table may be missing). Error: %s' % str(e)) + if _migrate_attempt == 1: + try: + from django.core.management import call_command + call_command('migrate', 'firewall', verbosity=0) + _log.CyberCPLogFileWriter.writeToFile('getBannedIPs: ran migrate firewall, retrying.') + except Exception as migrate_err: + _log.CyberCPLogFileWriter.writeToFile('getBannedIPs: migrate firewall failed: %s' % str(migrate_err)) + else: + _log.CyberCPLogFileWriter.writeToFile('getBannedIPs: DB read failed after retry, merging with JSON.') + if _migrate_attempt == 2: + break + except Exception as e: + import plogical.CyberCPLogFileWriter as _log + _log.CyberCPLogFileWriter.writeToFile('getBannedIPs: DB read failed, merging with JSON (%s)' % str(e)) + break # If ORM returned nothing but we have the table, try raw SQL as fallback if not active_banned_ips: diff --git a/install/install.py b/install/install.py index f4d7b2bce..495688bda 100644 --- a/install/install.py +++ b/install/install.py @@ -3455,6 +3455,11 @@ skip-ssl command = f"{python_path} {manage_py} migrate --noinput" preFlightsChecks.call(command, self.distro, command, command, 1, 1, os.EX_OSERR, True) + # Ensure firewall (banned IPs) table exists so /firewall/#banned-ips works + logging.InstallLog.writeToFile("Applying firewall migrations (firewall_bannedips)...") + command = f"{python_path} {manage_py} migrate firewall --noinput" + preFlightsChecks.call(command, self.distro, command, command, 1, 1, os.EX_OSERR, True) + logging.InstallLog.writeToFile("Django migrations completed successfully!") preFlightsChecks.stdOut("Django migrations completed successfully!") diff --git a/plogical/CyberPanelUpgrade.py b/plogical/CyberPanelUpgrade.py index df6abdac6..91fc2efb2 100644 --- a/plogical/CyberPanelUpgrade.py +++ b/plogical/CyberPanelUpgrade.py @@ -83,6 +83,7 @@ class UpgradeCyberPanel: Upgrade.s3BackupMigrations() Upgrade.containerMigrations() Upgrade.manageServiceMigrations() + Upgrade.firewallMigrations() self.PostStatus('Database updated.,55') diff --git a/plogical/upgrade.py b/plogical/upgrade.py index 0cfbc58f1..05038d017 100644 --- a/plogical/upgrade.py +++ b/plogical/upgrade.py @@ -2957,6 +2957,20 @@ CREATE TABLE `websiteFunctions_backupsv2` (`id` integer AUTO_INCREMENT NOT NULL except: pass + # Sync Django migration state so manage.py migrate sees dockerManager as applied + try: + cwd = os.getcwd() + os.chdir('/usr/local/CyberCP') + py = Upgrade._python_for_manage() + command = py + ' manage.py migrate dockerManager --noinput' + Upgrade.executioner(command, 'migrate dockerManager', 0) + os.chdir(cwd) + except Exception: + try: + os.chdir(cwd) + except Exception: + pass + @staticmethod def containerMigrations(): try: @@ -3173,6 +3187,79 @@ CREATE TABLE `websiteFunctions_backupsv2` (`id` integer AUTO_INCREMENT NOT NULL except: pass + @staticmethod + def firewallMigrations(): + """Ensure firewall app tables exist (e.g. firewall_bannedips for Ban IP). Upgrade does not run GeneralMigrations(), so run migrate firewall explicitly.""" + try: + cwd = os.getcwd() + os.chdir('/usr/local/CyberCP') + py = Upgrade._python_for_manage() + command = py + ' manage.py migrate firewall --noinput' + Upgrade.executioner(command, 'Run firewall migrations (firewall_bannedips)', 0) + os.chdir(cwd) + except Exception as e: + ErrorSanitizer.log_error_securely(e, 'firewallMigrations') + try: + os.chdir(cwd) + except Exception: + pass + Upgrade.syncBannedIPsJsonToDb() + + @staticmethod + def syncBannedIPsJsonToDb(): + """Sync banned IPs from JSON (e.g. from base dashboard Ban IP) into firewall_bannedips so Firewall > Banned IPs shows all.""" + try: + import json + for path in ['/usr/local/CyberCP/data/banned_ips.json', '/etc/cyberpanel/banned_ips.json']: + if not os.path.exists(path): + continue + try: + with open(path, 'r') as f: + data = json.load(f) + except Exception: + continue + if not isinstance(data, list): + continue + connection, cursor = Upgrade.setupConnection('cyberpanel') + if not cursor: + continue + try: + cursor.execute('SELECT id FROM loginSystem_administrator ORDER BY id ASC LIMIT 1') + row = cursor.fetchone() + admin_id = int(row[0]) if row else 1 + except Exception: + admin_id = 1 + for b in data: + if not b.get('active', True): + continue + ip_val = (b.get('ip') or '').strip() + if not ip_val or len(ip_val) > 45: + continue + reason = (b.get('reason') or 'Banned from dashboard')[:255] + banned_on = b.get('banned_on') + if isinstance(banned_on, (int, float)): + from_unixtime = banned_on + else: + from_unixtime = int(__import__('time').time()) + try: + cursor.execute( + """INSERT IGNORE INTO firewall_bannedips (ip_address, reason, duration, banned_on, expires, active, admin_id) + VALUES (%s, %s, 'permanent', FROM_UNIXTIME(%s), NULL, 1, %s)""", + (ip_val, reason, from_unixtime, admin_id) + ) + except Exception: + pass + try: + connection.close() + except Exception: + pass + break + except Exception as e: + try: + ErrorSanitizer.log_error_securely(e, 'syncBannedIPsJsonToDb') + except Exception: + pass + @staticmethod def _python_for_manage(): """Resolve Python for manage.py (avoid FileNotFoundError when /usr/local/CyberPanel/bin/python missing).""" @@ -3998,20 +4085,44 @@ class Migration(migrations.Migration): # Clone the new repository (use CYBERPANEL_GIT_USER for fork, e.g. master3395) git_user = os.environ.get('CYBERPANEL_GIT_USER', 'master3395') + upstream_user = os.environ.get('CYBERPANEL_UPSTREAM_GIT_USER', 'usmannasir') + checkout_ok = False + Upgrade.stdOut("Cloning fresh CyberPanel repository...") command = 'git clone https://github.com/%s/cyberpanel CyberCP' % git_user if not Upgrade.executioner(command, command, 1): - # Try to restore backup if clone fails Upgrade.stdOut("Clone failed, attempting to restore backup...") Upgrade.restoreCriticalFiles(backup_dir, backed_up_files) return 0, 'Failed to clone CyberPanel repository' - - # Checkout the correct branch + os.chdir('/usr/local/CyberCP') command = 'git checkout %s' % (branch) - if not Upgrade.executioner(command, command, 1): - Upgrade.stdOut(f"Warning: Failed to checkout branch {branch}, continuing with default branch") - + if Upgrade.executioner(command, command, 1): + checkout_ok = True + + if not checkout_ok and git_user != upstream_user: + Upgrade.stdOut("Branch not found on primary repo, trying upstream (%s)..." % upstream_user) + os.chdir('/usr/local') + if os.path.exists('CyberCP'): + try: + shutil.rmtree('CyberCP') + except Exception as e: + Upgrade.stdOut("Error removing CyberCP: %s" % str(e)) + Upgrade.restoreCriticalFiles(backup_dir, backed_up_files) + return 0, 'Failed to remove CyberCP for upstream clone' + command = 'git clone https://github.com/%s/cyberpanel CyberCP' % upstream_user + if not Upgrade.executioner(command, command, 1): + Upgrade.restoreCriticalFiles(backup_dir, backed_up_files) + return 0, 'Failed to clone upstream CyberPanel repository' + os.chdir('/usr/local/CyberCP') + command = 'git checkout %s' % (branch) + if Upgrade.executioner(command, command, 1): + checkout_ok = True + + if not checkout_ok: + Upgrade.restoreCriticalFiles(backup_dir, backed_up_files) + return 0, 'Branch %s not found on primary or upstream repo; ensure it exists.' % branch + # Restore all backed up configuration files (except settings.py) Upgrade.stdOut("Restoring configuration files...") Upgrade.restoreCriticalFiles(backup_dir, backed_up_files) @@ -5136,6 +5247,9 @@ echo $oConfig->Save() ? 'Done' : 'Error'; command = "mkdir -p /usr/local/lscp/cyberpanel/logs" Upgrade.executioner(command, 0) + command = "mkdir -p /usr/local/CyberCP/data" + Upgrade.executioner(command, 0) + @staticmethod def upgradeDovecot(): try: @@ -5933,6 +6047,7 @@ slowlog = /var/log/php{version}-fpm-slow.log Upgrade.s3BackupMigrations() Upgrade.containerMigrations() Upgrade.manageServiceMigrations() + Upgrade.firewallMigrations() Upgrade.enableServices() # Apply AlmaLinux 9 fixes before other installations