Docker containers 500 fix, firewall banned IPs, container logs readability, base Ban IP sync to DB

- dockerManager: add 0001_initial migration (CREATE TABLE IF NOT EXISTS), migrate-and-retry on DB errors, safe error response, fix logging.CyberCPLogFileWriter.writeToFile
- dockerManager/views: listContainersPage fallback HTML with error message if template fails
- dockerManager/viewContainer: improve container log readability (font-size 1rem, color #f1f5f9, line-height 1.6)
- baseTemplate: blockIPAddress also adds ban to firewall BannedIP model so Firewall > Banned IPs shows all
- firewall: getBannedIPs migrate-and-retry on OperationalError/ProgrammingError; install runs migrate firewall
- plogical/upgrade: syncBannedIPsJsonToDb() to sync JSON bans to firewall_bannedips; firewallMigrations() calls it; CyberPanelUpgrade runs firewallMigrations(); someDirectories creates /usr/local/CyberCP/data
- install: explicit migrate firewall after global migrate

baseTemplate/views.py

@@ -1454,6 +1454,27 @@ def blockIPAddress(request):
                     # Save to file
                     with open(primary_file, 'w') as f:
                         json.dump(banned_ips, f, indent=2)
+                    
+                    # Also add to firewall DB so it shows on Firewall > Banned IPs
+                    try:
+                        from firewall.models import BannedIP
+                        from django.utils import timezone
+                        user_id = request.session.get('userID')
+                        if user_id:
+                            admin = Administrator.objects.get(pk=user_id)
+                            BannedIP.objects.get_or_create(
+                                ip_address=ip_address,
+                                defaults={
+                                    'reason': reason,
+                                    'duration': 'permanent',
+                                    'banned_on': timezone.now(),
+                                    'expires': None,
+                                    'active': True,
+                                    'admin': admin,
+                                }
+                            )
+                    except Exception as db_e:
+                        logging.CyberCPLogFileWriter.writeToFile(f'Warning: Failed to add banned IP to firewall DB: {str(db_e)}')
             except Exception as e:
                 # Log but don't fail the request if JSON update fails
                 import plogical.CyberCPLogFileWriter as logging

dockerManager/container.py

@@ -16,7 +16,7 @@ import plogical.CyberCPLogFileWriter as logging
 from plogical.errorSanitizer import secure_error_response, secure_log_error
 from django.shortcuts import HttpResponse, render, redirect
 from django.urls import reverse
-from django.db.utils import OperationalError
+from django.db.utils import OperationalError, ProgrammingError, InternalError
 from loginSystem.models import Administrator
 import subprocess
 import shlex
@@ -258,32 +258,69 @@ class ContainerManager(multi.Thread):
                                                 "showUnlistedContainer": showUnlistedContainer}, 'admin')
             return proc.render()
 
-        try:
-            return _render_list()
-        except OperationalError as e:
-            logging.writeToFile(
-                "Docker containers list: DB error (table may be missing). Running migrations. Error: %s" % str(e)
+        def _run_migrate_and_retry(exc):
+            logging.CyberCPLogFileWriter.writeToFile(
+                "Docker containers list: DB error (table/column may be missing). Running migrations. Error: %s" % str(exc)
             )
             try:
+                # Ensure table exists: raw SQL path (idempotent) then Django migrate
+                try:
+                    from plogical.upgrade import Upgrade
+                    Upgrade.dockerMigrations()
+                except Exception as _:
+                    pass
                 from django.core.management import call_command
                 call_command('migrate', 'dockerManager', verbosity=0)
                 return _render_list()
             except Exception as migrate_err:
-                logging.writeToFile(
+                logging.CyberCPLogFileWriter.writeToFile(
                     "Docker containers list: migrate failed. Error: %s" % str(migrate_err)
                 )
+                return _safe_error_response(
+                    request,
+                    'Docker Manager database not ready. Please run upgrade or: manage.py migrate dockerManager',
+                    status=500
+                )
+
+        def _safe_error_response(request, message, status=500):
+            """Return error page or minimal HttpResponse if template render fails."""
+            try:
                 return render(
                     request,
                     'baseTemplate/error.html',
-                    {'error_message': 'Docker Manager database not ready. Please run upgrade or: manage.py migrate dockerManager'}
+                    {'error_message': message},
+                    status=status
                 )
+            except Exception as render_err:
+                logging.CyberCPLogFileWriter.writeToFile("Docker listContainers: render error.html failed: %s" % str(render_err))
+                safe_msg = (message or "Error")[:500].replace("<", "&lt;").replace(">", "&gt;")
+                html = "<!DOCTYPE html><html><body><h1>Docker Containers</h1><p>%s</p></body></html>" % safe_msg
+                return HttpResponse(html, status=status)
+
+        try:
+            return _render_list()
+        except OperationalError as e:
+            return _run_migrate_and_retry(e)
+        except ProgrammingError as e:
+            return _run_migrate_and_retry(e)
+        except InternalError as e:
+            return _run_migrate_and_retry(e)
         except Exception as e:
+            import traceback
             secure_log_error(e, 'docker_list_containers')
-            return render(
-                request,
-                'baseTemplate/error.html',
-                {'error_message': 'Containers list could not be loaded. Check error logs.'}
+            logging.CyberCPLogFileWriter.writeToFile(
+                "Docker containers list: %s: %s" % (type(e).__name__, str(e))
             )
+            logging.CyberCPLogFileWriter.writeToFile("Docker containers list traceback:\n%s" % traceback.format_exc())
+            # User-friendly message for common Docker errors
+            err_msg = str(e)
+            if hasattr(e, 'explicit') and getattr(e, 'explicit', None):
+                err_msg = getattr(e, 'explicit', err_msg) or err_msg
+            if 'docker' in type(e).__name__.lower() or 'connection' in err_msg.lower() or 'refused' in err_msg.lower():
+                message = 'Docker is not responding. Ensure the Docker daemon is running and try again. Error: %s' % (err_msg[:150])
+            else:
+                message = 'Containers list could not be loaded. Check error logs.'
+            return _safe_error_response(request, message, status=500)
 
     def getContainerLogs(self, userID=None, data=None):
         try:

dockerManager/migrations/0001_initial.py

@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*-
+# Docker Manager initial migration.
+# Creates dockerManager_containers table if not exists (safe when upgrade.dockerMigrations() already ran).
+
+from django.db import migrations
+
+
+def create_table_if_not_exists(apps, schema_editor):
+    """Create dockerManager_containers with full schema. Idempotent via IF NOT EXISTS."""
+    if schema_editor.connection.vendor != 'mysql':
+        return
+    # Use raw SQL so we can do IF NOT EXISTS; matches upgrade.dockerMigrations() final schema
+    sql = """
+    CREATE TABLE IF NOT EXISTS dockerManager_containers (
+        id int(11) NOT NULL AUTO_INCREMENT,
+        name varchar(150) NOT NULL,
+        cid varchar(64) NOT NULL DEFAULT '',
+        admin_id int(11) NOT NULL,
+        image varchar(50) NOT NULL DEFAULT 'unknown',
+        tag varchar(50) NOT NULL DEFAULT 'unknown',
+        memory int(11) NOT NULL DEFAULT 0,
+        ports longtext NOT NULL,
+        volumes longtext NOT NULL,
+        env longtext NOT NULL,
+        startOnReboot int(11) NOT NULL DEFAULT 0,
+        network varchar(100) NOT NULL DEFAULT 'bridge',
+        network_mode varchar(50) NOT NULL DEFAULT 'bridge',
+        extra_options longtext NOT NULL,
+        PRIMARY KEY (id),
+        UNIQUE KEY name (name),
+        KEY dockerManager_contai_admin_id_58fb62b7_fk_loginSyst (admin_id),
+        CONSTRAINT dockerManager_contai_admin_id_58fb62b7_fk_loginSyst
+            FOREIGN KEY (admin_id) REFERENCES loginSystem_administrator (id)
+    )
+    """
+    schema_editor.execute(sql)
+
+
+def noop_reverse(apps, schema_editor):
+    pass
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('loginSystem', '__first__'),
+    ]
+
+    operations = [
+        migrations.RunPython(create_table_if_not_exists, noop_reverse),
+    ]

dockerManager/templates/dockerManager/viewContainer.html

@@ -429,15 +429,16 @@
     }
     
     .terminal-content {
-        font-family: 'SF Mono', Monaco, Consolas, monospace;
-        font-size: 0.8125rem;
-        line-height: 1.5;
-        color: #e2e8f0;
+        font-family: 'SF Mono', Monaco, Consolas, 'Courier New', monospace;
+        font-size: 1rem;
+        line-height: 1.6;
+        color: #f1f5f9;
         background: #1a202c;
         padding: 1.5rem;
         height: 400px;
         overflow-y: auto;
         white-space: pre-wrap;
+        word-wrap: break-word;
     }
     
     .terminal-content::-webkit-scrollbar {

dockerManager/views.py

@@ -197,15 +197,32 @@ def listContainersPage(request):
     except KeyError:
         return redirect(loadLoginPage)
     except Exception as e:
+        import traceback
         from django.shortcuts import render
+        from django.http import HttpResponse
         from plogical.CyberCPLogFileWriter import CyberCPLogFileWriter as logging
-        logging.writeToFile("listContainersPage error: %s" % str(e))
-        return render(
-            request,
-            'baseTemplate/error.html',
-            {'error_message': 'Containers page could not be loaded. Check error logs.'},
-            status=500
-        )
+        err_msg = str(e)
+        err_type = type(e).__name__
+        logging.writeToFile("listContainersPage error: %s: %s" % (err_type, err_msg))
+        logging.writeToFile("listContainersPage traceback:\n%s" % traceback.format_exc())
+        # Try standard error template first; if it fails (e.g. missing), return minimal HTML so user sees the error
+        try:
+            return render(
+                request,
+                'baseTemplate/error.html',
+                {'error_message': 'Containers page could not be loaded. Check error logs.'},
+                status=500
+            )
+        except Exception as render_err:
+            logging.writeToFile("listContainersPage: render error.html failed: %s" % str(render_err))
+            safe_msg = err_type + ": " + (err_msg[:200] if err_msg else "Unknown error")
+            html = (
+                "<!DOCTYPE html><html><head><title>Docker Containers Error</title></head><body>"
+                "<h1>Containers page error</h1><p>%s</p>"
+                "<p>Check <code>/home/cyberpanel/error-logs.txt</code> for full traceback.</p>"
+                "</body></html>"
+            ) % (safe_msg.replace("<", "&lt;").replace(">", "&gt;"))
+            return HttpResponse(html, status=500)
 
 
 @preDockerRun

firewall/firewallManager.py

@@ -1924,37 +1924,55 @@ class FirewallManager:
             active_banned_ips = []
             db_ips = set()  # IPs already added from DB (for merge with JSON)
             current_time = int(time.time())
+            from django.db.utils import OperationalError, ProgrammingError
 
-            try:
-                from firewall.models import BannedIP
-                from django.db.models import Q
+            for _migrate_attempt in (1, 2):
+                try:
+                    from firewall.models import BannedIP
+                    from django.db.models import Q
 
-                banned_ips_queryset = BannedIP.objects.filter(
-                    active=True
-                ).filter(
-                    Q(expires__isnull=True) | Q(expires__gt=current_time)
-                ).order_by('-banned_on')
+                    banned_ips_queryset = BannedIP.objects.filter(
+                        active=True
+                    ).filter(
+                        Q(expires__isnull=True) | Q(expires__gt=current_time)
+                    ).order_by('-banned_on')
 
-                for banned_ip in banned_ips_queryset:
-                    try:
-                        ip_data = {
-                            'id': banned_ip.id,
-                            'ip': banned_ip.ip_address,
-                            'reason': banned_ip.reason,
-                            'duration': banned_ip.duration,
-                            'banned_on': banned_ip.get_banned_on_display(),
-                            'expires': banned_ip.get_expires_display(),
-                            'active': not banned_ip.is_expired() and banned_ip.active
-                        }
-                        if ip_data['active']:
-                            active_banned_ips.append(ip_data)
-                            db_ips.add(banned_ip.ip_address)
-                    except Exception as row_e:
-                        import plogical.CyberCPLogFileWriter as _log
-                        _log.CyberCPLogFileWriter.writeToFile('getBannedIPs: skip row %s: %s' % (getattr(banned_ip, 'ip_address', '?'), str(row_e)))
-            except Exception as e:
-                import plogical.CyberCPLogFileWriter as _log
-                _log.CyberCPLogFileWriter.writeToFile('getBannedIPs: DB read failed, merging with JSON (%s)' % str(e))
+                    for banned_ip in banned_ips_queryset:
+                        try:
+                            ip_data = {
+                                'id': banned_ip.id,
+                                'ip': banned_ip.ip_address,
+                                'reason': banned_ip.reason,
+                                'duration': banned_ip.duration,
+                                'banned_on': banned_ip.get_banned_on_display(),
+                                'expires': banned_ip.get_expires_display(),
+                                'active': not banned_ip.is_expired() and banned_ip.active
+                            }
+                            if ip_data['active']:
+                                active_banned_ips.append(ip_data)
+                                db_ips.add(banned_ip.ip_address)
+                        except Exception as row_e:
+                            import plogical.CyberCPLogFileWriter as _log
+                            _log.CyberCPLogFileWriter.writeToFile('getBannedIPs: skip row %s: %s' % (getattr(banned_ip, 'ip_address', '?'), str(row_e)))
+                    break
+                except (OperationalError, ProgrammingError) as e:
+                    import plogical.CyberCPLogFileWriter as _log
+                    _log.CyberCPLogFileWriter.writeToFile('getBannedIPs: DB error (table may be missing). Error: %s' % str(e))
+                    if _migrate_attempt == 1:
+                        try:
+                            from django.core.management import call_command
+                            call_command('migrate', 'firewall', verbosity=0)
+                            _log.CyberCPLogFileWriter.writeToFile('getBannedIPs: ran migrate firewall, retrying.')
+                        except Exception as migrate_err:
+                            _log.CyberCPLogFileWriter.writeToFile('getBannedIPs: migrate firewall failed: %s' % str(migrate_err))
+                    else:
+                        _log.CyberCPLogFileWriter.writeToFile('getBannedIPs: DB read failed after retry, merging with JSON.')
+                    if _migrate_attempt == 2:
+                        break
+                except Exception as e:
+                    import plogical.CyberCPLogFileWriter as _log
+                    _log.CyberCPLogFileWriter.writeToFile('getBannedIPs: DB read failed, merging with JSON (%s)' % str(e))
+                    break
 
             # If ORM returned nothing but we have the table, try raw SQL as fallback
             if not active_banned_ips:

install/install.py

@@ -3455,6 +3455,11 @@ skip-ssl
         command = f"{python_path} {manage_py} migrate --noinput"
         preFlightsChecks.call(command, self.distro, command, command, 1, 1, os.EX_OSERR, True)
 
+        # Ensure firewall (banned IPs) table exists so /firewall/#banned-ips works
+        logging.InstallLog.writeToFile("Applying firewall migrations (firewall_bannedips)...")
+        command = f"{python_path} {manage_py} migrate firewall --noinput"
+        preFlightsChecks.call(command, self.distro, command, command, 1, 1, os.EX_OSERR, True)
+
         logging.InstallLog.writeToFile("Django migrations completed successfully!")
         preFlightsChecks.stdOut("Django migrations completed successfully!")
 

plogical/CyberPanelUpgrade.py

@@ -83,6 +83,7 @@ class UpgradeCyberPanel:
         Upgrade.s3BackupMigrations()
         Upgrade.containerMigrations()
         Upgrade.manageServiceMigrations()
+        Upgrade.firewallMigrations()
 
         self.PostStatus('Database updated.,55')
 

plogical/upgrade.py

@@ -2957,6 +2957,20 @@ CREATE TABLE `websiteFunctions_backupsv2` (`id` integer AUTO_INCREMENT NOT NULL
         except:
             pass
 
+        # Sync Django migration state so manage.py migrate sees dockerManager as applied
+        try:
+            cwd = os.getcwd()
+            os.chdir('/usr/local/CyberCP')
+            py = Upgrade._python_for_manage()
+            command = py + ' manage.py migrate dockerManager --noinput'
+            Upgrade.executioner(command, 'migrate dockerManager', 0)
+            os.chdir(cwd)
+        except Exception:
+            try:
+                os.chdir(cwd)
+            except Exception:
+                pass
+
     @staticmethod
     def containerMigrations():
         try:
@@ -3173,6 +3187,79 @@ CREATE TABLE `websiteFunctions_backupsv2` (`id` integer AUTO_INCREMENT NOT NULL
         except:
             pass
 
+    @staticmethod
+    def firewallMigrations():
+        """Ensure firewall app tables exist (e.g. firewall_bannedips for Ban IP). Upgrade does not run GeneralMigrations(), so run migrate firewall explicitly."""
+        try:
+            cwd = os.getcwd()
+            os.chdir('/usr/local/CyberCP')
+            py = Upgrade._python_for_manage()
+            command = py + ' manage.py migrate firewall --noinput'
+            Upgrade.executioner(command, 'Run firewall migrations (firewall_bannedips)', 0)
+            os.chdir(cwd)
+        except Exception as e:
+            ErrorSanitizer.log_error_securely(e, 'firewallMigrations')
+            try:
+                os.chdir(cwd)
+            except Exception:
+                pass
+        Upgrade.syncBannedIPsJsonToDb()
+
+    @staticmethod
+    def syncBannedIPsJsonToDb():
+        """Sync banned IPs from JSON (e.g. from base dashboard Ban IP) into firewall_bannedips so Firewall > Banned IPs shows all."""
+        try:
+            import json
+            for path in ['/usr/local/CyberCP/data/banned_ips.json', '/etc/cyberpanel/banned_ips.json']:
+                if not os.path.exists(path):
+                    continue
+                try:
+                    with open(path, 'r') as f:
+                        data = json.load(f)
+                except Exception:
+                    continue
+                if not isinstance(data, list):
+                    continue
+                connection, cursor = Upgrade.setupConnection('cyberpanel')
+                if not cursor:
+                    continue
+                try:
+                    cursor.execute('SELECT id FROM loginSystem_administrator ORDER BY id ASC LIMIT 1')
+                    row = cursor.fetchone()
+                    admin_id = int(row[0]) if row else 1
+                except Exception:
+                    admin_id = 1
+                for b in data:
+                    if not b.get('active', True):
+                        continue
+                    ip_val = (b.get('ip') or '').strip()
+                    if not ip_val or len(ip_val) > 45:
+                        continue
+                    reason = (b.get('reason') or 'Banned from dashboard')[:255]
+                    banned_on = b.get('banned_on')
+                    if isinstance(banned_on, (int, float)):
+                        from_unixtime = banned_on
+                    else:
+                        from_unixtime = int(__import__('time').time())
+                    try:
+                        cursor.execute(
+                            """INSERT IGNORE INTO firewall_bannedips (ip_address, reason, duration, banned_on, expires, active, admin_id)
+                               VALUES (%s, %s, 'permanent', FROM_UNIXTIME(%s), NULL, 1, %s)""",
+                            (ip_val, reason, from_unixtime, admin_id)
+                        )
+                    except Exception:
+                        pass
+                try:
+                    connection.close()
+                except Exception:
+                    pass
+                break
+        except Exception as e:
+            try:
+                ErrorSanitizer.log_error_securely(e, 'syncBannedIPsJsonToDb')
+            except Exception:
+                pass
+
     @staticmethod
     def _python_for_manage():
         """Resolve Python for manage.py (avoid FileNotFoundError when /usr/local/CyberPanel/bin/python missing)."""
@@ -3998,20 +4085,44 @@ class Migration(migrations.Migration):
 
             # Clone the new repository (use CYBERPANEL_GIT_USER for fork, e.g. master3395)
             git_user = os.environ.get('CYBERPANEL_GIT_USER', 'master3395')
+            upstream_user = os.environ.get('CYBERPANEL_UPSTREAM_GIT_USER', 'usmannasir')
+            checkout_ok = False
+
             Upgrade.stdOut("Cloning fresh CyberPanel repository...")
             command = 'git clone https://github.com/%s/cyberpanel CyberCP' % git_user
             if not Upgrade.executioner(command, command, 1):
-                # Try to restore backup if clone fails
                 Upgrade.stdOut("Clone failed, attempting to restore backup...")
                 Upgrade.restoreCriticalFiles(backup_dir, backed_up_files)
                 return 0, 'Failed to clone CyberPanel repository'
-            
-            # Checkout the correct branch
+
             os.chdir('/usr/local/CyberCP')
             command = 'git checkout %s' % (branch)
-            if not Upgrade.executioner(command, command, 1):
-                Upgrade.stdOut(f"Warning: Failed to checkout branch {branch}, continuing with default branch")
-            
+            if Upgrade.executioner(command, command, 1):
+                checkout_ok = True
+
+            if not checkout_ok and git_user != upstream_user:
+                Upgrade.stdOut("Branch not found on primary repo, trying upstream (%s)..." % upstream_user)
+                os.chdir('/usr/local')
+                if os.path.exists('CyberCP'):
+                    try:
+                        shutil.rmtree('CyberCP')
+                    except Exception as e:
+                        Upgrade.stdOut("Error removing CyberCP: %s" % str(e))
+                        Upgrade.restoreCriticalFiles(backup_dir, backed_up_files)
+                        return 0, 'Failed to remove CyberCP for upstream clone'
+                command = 'git clone https://github.com/%s/cyberpanel CyberCP' % upstream_user
+                if not Upgrade.executioner(command, command, 1):
+                    Upgrade.restoreCriticalFiles(backup_dir, backed_up_files)
+                    return 0, 'Failed to clone upstream CyberPanel repository'
+                os.chdir('/usr/local/CyberCP')
+                command = 'git checkout %s' % (branch)
+                if Upgrade.executioner(command, command, 1):
+                    checkout_ok = True
+
+            if not checkout_ok:
+                Upgrade.restoreCriticalFiles(backup_dir, backed_up_files)
+                return 0, 'Branch %s not found on primary or upstream repo; ensure it exists.' % branch
+
             # Restore all backed up configuration files (except settings.py)
             Upgrade.stdOut("Restoring configuration files...")
             Upgrade.restoreCriticalFiles(backup_dir, backed_up_files)
@@ -5136,6 +5247,9 @@ echo $oConfig->Save() ? 'Done' : 'Error';
         command = "mkdir -p /usr/local/lscp/cyberpanel/logs"
         Upgrade.executioner(command, 0)
 
+        command = "mkdir -p /usr/local/CyberCP/data"
+        Upgrade.executioner(command, 0)
+
     @staticmethod
     def upgradeDovecot():
         try:
@@ -5933,6 +6047,7 @@ slowlog = /var/log/php{version}-fpm-slow.log
         Upgrade.s3BackupMigrations()
         Upgrade.containerMigrations()
         Upgrade.manageServiceMigrations()
+        Upgrade.firewallMigrations()
         Upgrade.enableServices()
 
         # Apply AlmaLinux 9 fixes before other installations