mirror of
https://github.com/myronblair/novacpx
synced 2026-06-30 17:50:41 -05:00
Add DKIM auto-provisioning, OS/panel self-update with self-healing
- AccountManager: auto-generate DKIM keypair + inject SPF/DKIM/DMARC DNS records on account create - AccountManager: rotateDKIM() method for key rotation with new selector - New dkim.php endpoint: list/view/rotate/provision DKIM keys per domain - schema.sql: add dkim_keys table - install.sh: install opendkim, wire into Postfix milter, fix dotfile copy (. vs *), fix config.ini permissions (root:www-data 640), copy VERSION to web root, add opendkim to service restart - api/index.php: fix NOVACPX_ROOT path (was 2 levels too high), fix CORS ports (8880-8883), VERSION fallback to /opt/novacpx-src - api/.htaccess: route all /api/* requests through index.php - system.php: check-os-update, apply-os-update (self-healing: auto-restart downed services, restore web root if panel ports go down), check-novacpx-update, apply-novacpx-update (PHP syntax validation before deploy, backup + restore on failure) - admin.js: Updates page now shows both NovaCPX panel updates and OS package upgrades in one section; sidebar badge shows combined count Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -369,4 +369,16 @@ INSERT INTO settings (`key`, `value`) VALUES
|
|||||||
('git_remote', 'https://github.com/myronblair/novacpx.git')
|
('git_remote', 'https://github.com/myronblair/novacpx.git')
|
||||||
ON DUPLICATE KEY UPDATE `value` = VALUES(`value`);
|
ON DUPLICATE KEY UPDATE `value` = VALUES(`value`);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS dkim_keys (
|
||||||
|
`id` INT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
|
||||||
|
`account_id` INT UNSIGNED NOT NULL,
|
||||||
|
`domain` VARCHAR(253) NOT NULL,
|
||||||
|
`selector` VARCHAR(63) NOT NULL DEFAULT 'mail',
|
||||||
|
`public_key` TEXT NOT NULL,
|
||||||
|
`private_key_path` VARCHAR(500) NOT NULL,
|
||||||
|
`created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
UNIQUE KEY uq_domain (domain),
|
||||||
|
CONSTRAINT fk_dkim_acct FOREIGN KEY (account_id) REFERENCES accounts(id) ON DELETE CASCADE
|
||||||
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||||
|
|
||||||
SET foreign_key_checks = 1;
|
SET foreign_key_checks = 1;
|
||||||
|
|||||||
+27
-3
@@ -364,6 +364,28 @@ apt-get install -y -qq proftpd-basic proftpd-mod-mysql >> "$LOG" 2>&1
|
|||||||
systemctl enable proftpd >> "$LOG" 2>&1
|
systemctl enable proftpd >> "$LOG" 2>&1
|
||||||
log "ProFTPD installed"
|
log "ProFTPD installed"
|
||||||
|
|
||||||
|
# ── OpenDKIM ─────────────────────────────────────────────────────────────────
|
||||||
|
step "Installing OpenDKIM"
|
||||||
|
apt-get install -y -qq opendkim opendkim-tools >> "$LOG" 2>&1
|
||||||
|
mkdir -p /etc/opendkim/keys
|
||||||
|
cat >> /etc/opendkim/opendkim.conf <<DKIM
|
||||||
|
Mode sv
|
||||||
|
Canonicalization relaxed/simple
|
||||||
|
KeyTable /etc/opendkim/key.table
|
||||||
|
SigningTable refile:/etc/opendkim/signing.table
|
||||||
|
ExternalIgnoreList refile:/etc/opendkim/trusted.hosts
|
||||||
|
InternalHosts refile:/etc/opendkim/trusted.hosts
|
||||||
|
DKIM
|
||||||
|
touch /etc/opendkim/key.table /etc/opendkim/signing.table
|
||||||
|
echo "127.0.0.1\nlocalhost" > /etc/opendkim/trusted.hosts
|
||||||
|
chown -R opendkim:opendkim /etc/opendkim
|
||||||
|
# Wire opendkim into Postfix
|
||||||
|
postconf -e "milter_default_action = accept" >> "$LOG" 2>&1
|
||||||
|
postconf -e "smtpd_milters = local:/run/opendkim/opendkim.sock" >> "$LOG" 2>&1
|
||||||
|
postconf -e "non_smtpd_milters = local:/run/opendkim/opendkim.sock" >> "$LOG" 2>&1
|
||||||
|
systemctl enable opendkim >> "$LOG" 2>&1
|
||||||
|
log "OpenDKIM installed"
|
||||||
|
|
||||||
# ── SSL Certificate ───────────────────────────────────────────────────────────
|
# ── SSL Certificate ───────────────────────────────────────────────────────────
|
||||||
step "Generating Self-Signed SSL (Panel)"
|
step "Generating Self-Signed SSL (Panel)"
|
||||||
mkdir -p /etc/novacpx/ssl
|
mkdir -p /etc/novacpx/ssl
|
||||||
@@ -452,10 +474,11 @@ mkdir -p "$WEB_ROOT" "$PANEL_DIR"
|
|||||||
|
|
||||||
# Install panel files from GitHub
|
# Install panel files from GitHub
|
||||||
if [[ -d /opt/novacpx-src ]]; then
|
if [[ -d /opt/novacpx-src ]]; then
|
||||||
cp -r /opt/novacpx-src/panel/public/* "$WEB_ROOT/"
|
cp -r /opt/novacpx-src/panel/public/. "$WEB_ROOT/"
|
||||||
cp -r /opt/novacpx-src/panel/api "$WEB_ROOT/api"
|
cp -r /opt/novacpx-src/panel/api "$WEB_ROOT/api"
|
||||||
cp -r /opt/novacpx-src/panel/lib "$WEB_ROOT/lib"
|
cp -r /opt/novacpx-src/panel/lib "$WEB_ROOT/lib"
|
||||||
cp -r /opt/novacpx-src/panel/lib /opt/novacpx/lib
|
cp -r /opt/novacpx-src/panel/lib /opt/novacpx/lib
|
||||||
|
cp /opt/novacpx-src/VERSION "$WEB_ROOT/VERSION" 2>/dev/null || true
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Write config
|
# Write config
|
||||||
@@ -480,7 +503,8 @@ version = ${NOVACPX_VERSION}
|
|||||||
server = ${WEB_SERVER}
|
server = ${WEB_SERVER}
|
||||||
php_default = ${PHP_DEFAULT}
|
php_default = ${PHP_DEFAULT}
|
||||||
CONFIG
|
CONFIG
|
||||||
chmod 600 /etc/novacpx/config.ini
|
chown root:www-data /etc/novacpx/config.ini
|
||||||
|
chmod 640 /etc/novacpx/config.ini
|
||||||
|
|
||||||
# Import database schema
|
# Import database schema
|
||||||
if [[ -f /opt/novacpx-src/db/schema.sql ]]; then
|
if [[ -f /opt/novacpx-src/db/schema.sql ]]; then
|
||||||
@@ -580,7 +604,7 @@ else
|
|||||||
systemctl restart apache2 >> "$LOG" 2>&1
|
systemctl restart apache2 >> "$LOG" 2>&1
|
||||||
fi
|
fi
|
||||||
$INSTALL_MYSQL && systemctl restart mysql >> "$LOG" 2>&1
|
$INSTALL_MYSQL && systemctl restart mysql >> "$LOG" 2>&1
|
||||||
systemctl restart postfix dovecot proftpd named >> "$LOG" 2>&1
|
systemctl restart postfix dovecot proftpd named opendkim >> "$LOG" 2>&1
|
||||||
log "All services started"
|
log "All services started"
|
||||||
|
|
||||||
# ── Done ─────────────────────────────────────────────────────────────────────
|
# ── Done ─────────────────────────────────────────────────────────────────────
|
||||||
|
|||||||
@@ -0,0 +1,87 @@
|
|||||||
|
<?php
|
||||||
|
/**
|
||||||
|
* DKIM key management endpoint
|
||||||
|
* Actions: list, rotate, view
|
||||||
|
*/
|
||||||
|
|
||||||
|
require_once NOVACPX_LIB . '/AccountManager.php';
|
||||||
|
require_once NOVACPX_LIB . '/DNSManager.php';
|
||||||
|
|
||||||
|
$db = DB::getInstance();
|
||||||
|
$user = $currentUser;
|
||||||
|
|
||||||
|
switch ($action) {
|
||||||
|
|
||||||
|
case 'list':
|
||||||
|
// Admin: all domains. User/reseller: their own accounts
|
||||||
|
if ($user['role'] === 'admin') {
|
||||||
|
$keys = $db->fetchAll(
|
||||||
|
"SELECT dk.*, a.username FROM dkim_keys dk JOIN accounts a ON dk.account_id = a.id ORDER BY dk.domain"
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
$keys = $db->fetchAll(
|
||||||
|
"SELECT dk.* FROM dkim_keys dk
|
||||||
|
JOIN accounts a ON dk.account_id = a.id
|
||||||
|
WHERE a.user_id = ?
|
||||||
|
ORDER BY dk.domain",
|
||||||
|
[$user['id']]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
foreach ($keys as &$k) { unset($k['private_key_path']); }
|
||||||
|
Response::json(['keys' => $keys]);
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 'view':
|
||||||
|
$domain = trim($body['domain'] ?? '');
|
||||||
|
if (!$domain) Response::error('domain required', 400);
|
||||||
|
|
||||||
|
$row = $db->fetchOne("SELECT * FROM dkim_keys WHERE domain = ?", [$domain]);
|
||||||
|
if (!$row) Response::error('No DKIM key found for domain', 404);
|
||||||
|
|
||||||
|
// Access control
|
||||||
|
if ($user['role'] !== 'admin') {
|
||||||
|
$acct = $db->fetchOne("SELECT id FROM accounts WHERE id = ? AND user_id = ?", [$row['account_id'], $user['id']]);
|
||||||
|
if (!$acct) Response::error('Forbidden', 403);
|
||||||
|
}
|
||||||
|
|
||||||
|
unset($row['private_key_path']);
|
||||||
|
// Also return the full DNS TXT value
|
||||||
|
$row['dns_record_name'] = "mail._domainkey.{$domain}";
|
||||||
|
$row['dns_record_value'] = "v=DKIM1; k=rsa; p={$row['public_key']}";
|
||||||
|
Response::json(['key' => $row]);
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 'rotate':
|
||||||
|
$domain = trim($body['domain'] ?? '');
|
||||||
|
if (!$domain) Response::error('domain required', 400);
|
||||||
|
|
||||||
|
$acct = $db->fetchOne("SELECT a.id, a.user_id FROM accounts a JOIN domains d ON d.account_id = a.id WHERE d.domain = ? AND d.type = 'main'", [$domain]);
|
||||||
|
if (!$acct) Response::error('Domain not found', 404);
|
||||||
|
|
||||||
|
if ($user['role'] !== 'admin' && (int)$acct['user_id'] !== (int)$user['id']) {
|
||||||
|
Response::error('Forbidden', 403);
|
||||||
|
}
|
||||||
|
|
||||||
|
$selector = AccountManager::rotateDKIM((int)$acct['id'], $domain);
|
||||||
|
Response::json(['ok' => true, 'selector' => $selector, 'message' => "DKIM rotated. New selector: {$selector}._domainkey.{$domain}"]);
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 'provision':
|
||||||
|
// Re-provision SPF/DKIM/DMARC for a domain (admin only or own account)
|
||||||
|
$domain = trim($body['domain'] ?? '');
|
||||||
|
if (!$domain) Response::error('domain required', 400);
|
||||||
|
|
||||||
|
$acct = $db->fetchOne("SELECT a.id, a.user_id FROM accounts a JOIN domains d ON d.account_id = a.id WHERE d.domain = ?", [$domain]);
|
||||||
|
if (!$acct) Response::error('Domain not found', 404);
|
||||||
|
|
||||||
|
if ($user['role'] !== 'admin' && (int)$acct['user_id'] !== (int)$user['id']) {
|
||||||
|
Response::error('Forbidden', 403);
|
||||||
|
}
|
||||||
|
|
||||||
|
AccountManager::provisionEmailDNS((int)$acct['id'], $domain);
|
||||||
|
Response::json(['ok' => true, 'message' => "SPF, DKIM, and DMARC records provisioned for {$domain}"]);
|
||||||
|
break;
|
||||||
|
|
||||||
|
default:
|
||||||
|
Response::error("Unknown action: $action", 404);
|
||||||
|
}
|
||||||
@@ -88,6 +88,201 @@ match ($action) {
|
|||||||
]);
|
]);
|
||||||
})(),
|
})(),
|
||||||
|
|
||||||
|
// ── Check OS updates ─────────────────────────────────────────────────────
|
||||||
|
'check-os-update' => (function() use ($db) {
|
||||||
|
Auth::getInstance()->require('admin');
|
||||||
|
shell_exec('apt-get update -qq 2>/dev/null');
|
||||||
|
$out = shell_exec('apt-get -s upgrade 2>/dev/null | grep "^Inst " | head -50') ?: '';
|
||||||
|
$packages = array_values(array_filter(array_map(function($line) {
|
||||||
|
if (preg_match('/^Inst (\S+).*\[(\S+)\].*\((\S+)/', $line, $m)) {
|
||||||
|
return ['name' => $m[1], 'from' => $m[2], 'to' => $m[3]];
|
||||||
|
} elseif (preg_match('/^Inst (\S+)\s+\((\S+)/', $line, $m)) {
|
||||||
|
return ['name' => $m[1], 'from' => '', 'to' => $m[2]];
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}, explode("\n", trim($out)))));
|
||||||
|
$security = array_filter($packages, fn($p) => str_contains($p['name'] ?? '', 'security') ||
|
||||||
|
(bool)shell_exec("apt-get -s upgrade 2>/dev/null | grep -c \"^Inst {$p['name']}.*security\" 2>/dev/null"));
|
||||||
|
Response::success([
|
||||||
|
'upgradable' => count($packages),
|
||||||
|
'security_updates' => count($security),
|
||||||
|
'packages' => $packages,
|
||||||
|
'last_checked' => date('Y-m-d H:i:s'),
|
||||||
|
]);
|
||||||
|
})(),
|
||||||
|
|
||||||
|
// ── Apply OS update ───────────────────────────────────────────────────────
|
||||||
|
'apply-os-update' => (function() use ($db) {
|
||||||
|
Auth::getInstance()->require('admin');
|
||||||
|
|
||||||
|
$panelPorts = [PORT_USER, PORT_RESELLER, PORT_ADMIN];
|
||||||
|
$webSvc = defined('WEB_SERVER') && WEB_SERVER === 'nginx' ? 'nginx' : 'apache2';
|
||||||
|
|
||||||
|
// Snapshot service states before upgrade
|
||||||
|
$beforeServices = [];
|
||||||
|
foreach ([$webSvc, 'mysql', 'postfix', 'dovecot', 'proftpd', 'named'] as $svc) {
|
||||||
|
$beforeServices[$svc] = trim(shell_exec("systemctl is-active $svc 2>/dev/null") ?: 'unknown');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Backup panel web root
|
||||||
|
$backupDir = '/var/novacpx/backups/pre-os-update-' . date('YmdHis');
|
||||||
|
$webRoot = defined('WEB_ROOT') ? WEB_ROOT : '/srv/novacpx/public';
|
||||||
|
shell_exec("mkdir -p " . escapeshellarg($backupDir));
|
||||||
|
shell_exec("cp -a " . escapeshellarg($webRoot) . " " . escapeshellarg("$backupDir/public") . " 2>&1");
|
||||||
|
|
||||||
|
// Run upgrade (non-interactive, hold back kernel packages to avoid reboot surprise)
|
||||||
|
$env = 'DEBIAN_FRONTEND=noninteractive';
|
||||||
|
$opts = '-o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold"';
|
||||||
|
$out = shell_exec("$env apt-get upgrade -y -q $opts 2>&1");
|
||||||
|
|
||||||
|
// Self-healing: restart any service that went down
|
||||||
|
$healed = [];
|
||||||
|
sleep(3);
|
||||||
|
foreach ($beforeServices as $svc => $wasBefore) {
|
||||||
|
if ($wasBefore !== 'active') continue;
|
||||||
|
$nowState = trim(shell_exec("systemctl is-active $svc 2>/dev/null") ?: '');
|
||||||
|
if ($nowState !== 'active') {
|
||||||
|
shell_exec("systemctl restart $svc 2>/dev/null");
|
||||||
|
sleep(2);
|
||||||
|
$afterHeal = trim(shell_exec("systemctl is-active $svc 2>/dev/null") ?: '');
|
||||||
|
$healed[$svc] = $afterHeal === 'active' ? 'restarted' : 'FAILED';
|
||||||
|
if ($afterHeal !== 'active') {
|
||||||
|
novacpx_log('error', "Self-heal FAILED for $svc after OS upgrade");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify panel ports respond
|
||||||
|
$panelOk = [];
|
||||||
|
foreach ($panelPorts as $port) {
|
||||||
|
$resp = @fsockopen('127.0.0.1', $port, $errno, $errstr, 3);
|
||||||
|
$panelOk[$port] = (bool)$resp;
|
||||||
|
if ($resp) fclose($resp);
|
||||||
|
}
|
||||||
|
$panelDown = array_keys(array_filter($panelOk, fn($ok) => !$ok));
|
||||||
|
|
||||||
|
// If panel ports down, restore from backup and restart web server
|
||||||
|
if ($panelDown) {
|
||||||
|
shell_exec("cp -a " . escapeshellarg("$backupDir/public") . " " . escapeshellarg($webRoot) . " 2>&1");
|
||||||
|
shell_exec("systemctl restart $webSvc 2>/dev/null");
|
||||||
|
novacpx_log('error', 'Panel ports down after OS upgrade — restored from backup');
|
||||||
|
}
|
||||||
|
|
||||||
|
audit('system.os-update', "upgraded; healed:" . implode(',', array_keys($healed)));
|
||||||
|
Response::success([
|
||||||
|
'upgraded' => true,
|
||||||
|
'panel_ports_ok' => empty($panelDown),
|
||||||
|
'panel_ports_down' => $panelDown,
|
||||||
|
'services_healed' => $healed,
|
||||||
|
'backup_path' => $backupDir,
|
||||||
|
'upgrade_output' => substr($out ?: '', -2000),
|
||||||
|
]);
|
||||||
|
})(),
|
||||||
|
|
||||||
|
// ── Check NovaCPX update ─────────────────────────────────────────────────
|
||||||
|
'check-novacpx-update' => (function() use ($db) {
|
||||||
|
Auth::getInstance()->require('admin');
|
||||||
|
$srcDir = '/opt/novacpx-src';
|
||||||
|
if (!is_dir($srcDir)) Response::error('Source repo not found at /opt/novacpx-src');
|
||||||
|
$out = shell_exec("git -C " . escapeshellarg($srcDir) . " fetch origin 2>&1 && git -C " . escapeshellarg($srcDir) . " log HEAD..origin/main --oneline 2>/dev/null");
|
||||||
|
$updates = array_values(array_filter(explode("\n", trim($out ?: ''))));
|
||||||
|
$branch = trim(shell_exec("git -C " . escapeshellarg($srcDir) . " branch --show-current 2>/dev/null") ?: 'main');
|
||||||
|
$commit = trim(shell_exec("git -C " . escapeshellarg($srcDir) . " rev-parse --short HEAD 2>/dev/null") ?: '');
|
||||||
|
Response::success([
|
||||||
|
'updates_available' => count($updates),
|
||||||
|
'current_commit' => $commit,
|
||||||
|
'branch' => $branch,
|
||||||
|
'commits' => $updates,
|
||||||
|
]);
|
||||||
|
})(),
|
||||||
|
|
||||||
|
// ── Apply NovaCPX update ─────────────────────────────────────────────────
|
||||||
|
'apply-novacpx-update' => (function() use ($db) {
|
||||||
|
Auth::getInstance()->require('admin');
|
||||||
|
$srcDir = '/opt/novacpx-src';
|
||||||
|
$webRoot = defined('WEB_ROOT') ? WEB_ROOT : '/srv/novacpx/public';
|
||||||
|
$webSvc = defined('WEB_SERVER') && WEB_SERVER === 'nginx' ? 'nginx' : 'apache2';
|
||||||
|
|
||||||
|
if (!is_dir($srcDir)) Response::error('Source repo not found at /opt/novacpx-src');
|
||||||
|
|
||||||
|
$before = trim(shell_exec("git -C " . escapeshellarg($srcDir) . " rev-parse HEAD 2>/dev/null") ?: '');
|
||||||
|
|
||||||
|
// Backup current web root
|
||||||
|
$backupDir = '/var/novacpx/backups/pre-novacpx-update-' . date('YmdHis');
|
||||||
|
shell_exec("mkdir -p " . escapeshellarg($backupDir));
|
||||||
|
shell_exec("cp -a " . escapeshellarg($webRoot) . " " . escapeshellarg("$backupDir/public") . " 2>&1");
|
||||||
|
|
||||||
|
// Pull new code
|
||||||
|
$pull = shell_exec("git -C " . escapeshellarg($srcDir) . " pull origin main 2>&1");
|
||||||
|
$after = trim(shell_exec("git -C " . escapeshellarg($srcDir) . " rev-parse HEAD 2>/dev/null") ?: '');
|
||||||
|
$changed = $before !== $after;
|
||||||
|
|
||||||
|
if ($changed) {
|
||||||
|
// Validate PHP syntax before deploying
|
||||||
|
$phpFiles = glob($srcDir . '/panel/**/*.php', GLOB_BRACE) ?: [];
|
||||||
|
$syntaxErr = [];
|
||||||
|
foreach ($phpFiles as $f) {
|
||||||
|
$check = shell_exec("php -l " . escapeshellarg($f) . " 2>&1");
|
||||||
|
if (!str_contains($check, 'No syntax errors')) {
|
||||||
|
$syntaxErr[] = basename($f) . ': ' . trim($check);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($syntaxErr) {
|
||||||
|
// Syntax errors — abort, restore
|
||||||
|
shell_exec("git -C " . escapeshellarg($srcDir) . " reset --hard " . escapeshellarg($before) . " 2>&1");
|
||||||
|
Response::error('Update aborted — PHP syntax errors: ' . implode('; ', $syntaxErr));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Deploy files to web root
|
||||||
|
shell_exec("rsync -a --delete " . escapeshellarg("$srcDir/panel/public/") . " " . escapeshellarg("$webRoot/") . " 2>&1");
|
||||||
|
shell_exec("rsync -a " . escapeshellarg("$srcDir/panel/lib/") . " " . escapeshellarg("$webRoot/lib/") . " 2>&1");
|
||||||
|
shell_exec("rsync -a " . escapeshellarg("$srcDir/panel/api/") . " " . escapeshellarg("$webRoot/api/") . " 2>&1");
|
||||||
|
shell_exec("cp " . escapeshellarg("$srcDir/VERSION") . " " . escapeshellarg("$webRoot/VERSION") . " 2>/dev/null");
|
||||||
|
shell_exec("chown -R www-data:www-data " . escapeshellarg($webRoot));
|
||||||
|
|
||||||
|
// Run pending DB migrations
|
||||||
|
$migrDir = "$srcDir/db/migrations";
|
||||||
|
if (is_dir($migrDir)) {
|
||||||
|
foreach (glob("$migrDir/*.sql") as $sql) {
|
||||||
|
$migName = basename($sql, '.sql');
|
||||||
|
$already = $db->fetchOne("SELECT 1 FROM settings WHERE `key` = ?", ["migration_$migName"]);
|
||||||
|
if (!$already) {
|
||||||
|
$db->pdo()->exec(file_get_contents($sql));
|
||||||
|
$db->execute("INSERT INTO settings (`key`,`value`) VALUES (?,NOW()) ON DUPLICATE KEY UPDATE `value`=NOW()", ["migration_$migName"]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reload PHP-FPM to pick up new code
|
||||||
|
shell_exec("systemctl reload php8.3-fpm 2>/dev/null || true");
|
||||||
|
|
||||||
|
// Verify panel is still up
|
||||||
|
sleep(2);
|
||||||
|
$panelOk = @fsockopen('127.0.0.1', PORT_ADMIN, $e, $es, 3);
|
||||||
|
if (!$panelOk) {
|
||||||
|
// Restore backup and reload
|
||||||
|
shell_exec("rsync -a --delete " . escapeshellarg("$backupDir/public/") . " " . escapeshellarg("$webRoot/") . " 2>&1");
|
||||||
|
shell_exec("systemctl reload $webSvc 2>/dev/null");
|
||||||
|
novacpx_log('error', "NovaCPX update failed — panel down after deploy; restored from backup");
|
||||||
|
Response::error('Update deployed but panel went down — auto-restored from backup. Check logs.');
|
||||||
|
} else {
|
||||||
|
fclose($panelOk);
|
||||||
|
}
|
||||||
|
|
||||||
|
audit('system.novacpx-update', "novacpx:$before→$after");
|
||||||
|
novacpx_log('info', "NovaCPX updated $before → $after");
|
||||||
|
}
|
||||||
|
|
||||||
|
Response::success([
|
||||||
|
'updated' => $changed,
|
||||||
|
'from_commit' => $before,
|
||||||
|
'to_commit' => $after,
|
||||||
|
'pull_output' => $pull,
|
||||||
|
'backup_path' => $backupDir,
|
||||||
|
]);
|
||||||
|
})(),
|
||||||
|
|
||||||
// ── Server Stats ──────────────────────────────────────────────────────────
|
// ── Server Stats ──────────────────────────────────────────────────────────
|
||||||
'stats' => (function() use ($db) {
|
'stats' => (function() use ($db) {
|
||||||
// CPU/load
|
// CPU/load
|
||||||
|
|||||||
+8
-5
@@ -4,16 +4,19 @@
|
|||||||
* All requests: /api/{endpoint}/{action}
|
* All requests: /api/{endpoint}/{action}
|
||||||
*/
|
*/
|
||||||
|
|
||||||
define('NOVACPX_ROOT', dirname(__DIR__, 2));
|
define('NOVACPX_ROOT', dirname(__DIR__));
|
||||||
define('NOVACPX_API', __DIR__);
|
define('NOVACPX_API', __DIR__);
|
||||||
define('NOVACPX_LIB', NOVACPX_ROOT . '/panel/lib');
|
define('NOVACPX_LIB', NOVACPX_ROOT . '/lib');
|
||||||
|
|
||||||
header('Content-Type: application/json');
|
header('Content-Type: application/json');
|
||||||
header('X-NovaCPX-Version: ' . (file_get_contents(NOVACPX_ROOT . '/VERSION') ?: '1.0.0'));
|
$_ver = file_get_contents(NOVACPX_ROOT . '/VERSION')
|
||||||
|
?: file_get_contents('/opt/novacpx-src/VERSION')
|
||||||
|
?: '1.0.0';
|
||||||
|
header('X-NovaCPX-Version: ' . trim($_ver));
|
||||||
|
|
||||||
// CORS for same-origin panel requests
|
// CORS for same-origin panel requests (ports 8880/8881/8882/8883)
|
||||||
$origin = $_SERVER['HTTP_ORIGIN'] ?? '';
|
$origin = $_SERVER['HTTP_ORIGIN'] ?? '';
|
||||||
if (preg_match('#^https?://[^/]+:2083$#', $origin)) {
|
if (preg_match('#^https?://[^/]+:(888[0-3])$#', $origin)) {
|
||||||
header("Access-Control-Allow-Origin: $origin");
|
header("Access-Control-Allow-Origin: $origin");
|
||||||
header('Access-Control-Allow-Credentials: true');
|
header('Access-Control-Allow-Credentials: true');
|
||||||
header('Access-Control-Allow-Methods: GET, POST, PUT, DELETE, OPTIONS');
|
header('Access-Control-Allow-Methods: GET, POST, PUT, DELETE, OPTIONS');
|
||||||
|
|||||||
@@ -55,6 +55,9 @@ class AccountManager {
|
|||||||
// Create DNS zone
|
// Create DNS zone
|
||||||
DNSManager::createZone($acctId, $domain);
|
DNSManager::createZone($acctId, $domain);
|
||||||
|
|
||||||
|
// Auto-provision SPF, DKIM, DMARC records
|
||||||
|
self::provisionEmailDNS($acctId, $domain);
|
||||||
|
|
||||||
// Create PHP-FPM pool
|
// Create PHP-FPM pool
|
||||||
PHPManager::createPool($username, $phpVer);
|
PHPManager::createPool($username, $phpVer);
|
||||||
|
|
||||||
@@ -110,6 +113,72 @@ class AccountManager {
|
|||||||
novacpx_log('info', "Account terminated: {$acct['username']}");
|
novacpx_log('info', "Account terminated: {$acct['username']}");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static function provisionEmailDNS(int $acctId, string $domain): void {
|
||||||
|
// Generate DKIM keypair
|
||||||
|
$keyDir = "/etc/opendkim/keys/{$domain}";
|
||||||
|
self::shell("mkdir -p " . escapeshellarg($keyDir));
|
||||||
|
self::shell("opendkim-genkey -b 2048 -s mail -d " . escapeshellarg($domain) . " -D " . escapeshellarg($keyDir));
|
||||||
|
self::shell("chown -R opendkim:opendkim " . escapeshellarg($keyDir));
|
||||||
|
|
||||||
|
// Parse public key from .txt file
|
||||||
|
$keyTxt = @file_get_contents("{$keyDir}/mail.txt") ?: '';
|
||||||
|
preg_match('/p=([A-Za-z0-9+\/=]+)/', $keyTxt, $m);
|
||||||
|
$pubKey = $m[1] ?? '';
|
||||||
|
|
||||||
|
if ($pubKey) {
|
||||||
|
// Register domain/key in opendkim tables
|
||||||
|
self::shell("grep -q " . escapeshellarg($domain) . " /etc/opendkim/signing.table 2>/dev/null || echo " . escapeshellarg("*@{$domain} {$domain}") . " >> /etc/opendkim/signing.table");
|
||||||
|
self::shell("grep -q " . escapeshellarg($domain) . " /etc/opendkim/key.table 2>/dev/null || echo " . escapeshellarg("{$domain} {$domain}:mail:{$keyDir}/mail.private") . " >> /etc/opendkim/key.table");
|
||||||
|
self::shell("systemctl reload opendkim 2>/dev/null || true");
|
||||||
|
|
||||||
|
// Store in DB
|
||||||
|
$db = DB::getInstance();
|
||||||
|
$db->execute(
|
||||||
|
"INSERT INTO dkim_keys (account_id, domain, selector, public_key, private_key_path, created_at) VALUES (?,?,?,?,?,NOW()) ON DUPLICATE KEY UPDATE public_key=VALUES(public_key)",
|
||||||
|
[$acctId, $domain, 'mail', $pubKey, "{$keyDir}/mail.private"]
|
||||||
|
);
|
||||||
|
|
||||||
|
// DKIM TXT record
|
||||||
|
DNSManager::addRecord($acctId, $domain, 'TXT', "mail._domainkey", "v=DKIM1; k=rsa; p={$pubKey}", 300);
|
||||||
|
}
|
||||||
|
|
||||||
|
// SPF
|
||||||
|
DNSManager::addRecord($acctId, $domain, 'TXT', '@', "v=spf1 mx a ~all", 300);
|
||||||
|
// DMARC
|
||||||
|
DNSManager::addRecord($acctId, $domain, 'TXT', '_dmarc', "v=DMARC1; p=quarantine; rua=mailto:dmarc@{$domain}", 300);
|
||||||
|
|
||||||
|
novacpx_log('info', "Email DNS provisioned for $domain");
|
||||||
|
}
|
||||||
|
|
||||||
|
public static function rotateDKIM(int $acctId, string $domain): string {
|
||||||
|
$db = DB::getInstance();
|
||||||
|
$selector = 'mail' . date('Ym');
|
||||||
|
$keyDir = "/etc/opendkim/keys/{$domain}";
|
||||||
|
self::shell("mkdir -p " . escapeshellarg($keyDir));
|
||||||
|
self::shell("opendkim-genkey -b 2048 -s {$selector} -d " . escapeshellarg($domain) . " -D " . escapeshellarg($keyDir));
|
||||||
|
self::shell("chown -R opendkim:opendkim " . escapeshellarg($keyDir));
|
||||||
|
|
||||||
|
$keyTxt = @file_get_contents("{$keyDir}/{$selector}.txt") ?: '';
|
||||||
|
preg_match('/p=([A-Za-z0-9+\/=]+)/', $keyTxt, $m);
|
||||||
|
$pubKey = $m[1] ?? '';
|
||||||
|
if (!$pubKey) throw new RuntimeException("DKIM key generation failed");
|
||||||
|
|
||||||
|
// Update key.table
|
||||||
|
$keyTableLine = "{$domain} {$domain}:{$selector}:{$keyDir}/{$selector}.private";
|
||||||
|
self::shell("sed -i " . escapeshellarg("/^{$domain} /d") . " /etc/opendkim/key.table 2>/dev/null; echo " . escapeshellarg($keyTableLine) . " >> /etc/opendkim/key.table");
|
||||||
|
self::shell("systemctl reload opendkim 2>/dev/null || true");
|
||||||
|
|
||||||
|
$db->execute(
|
||||||
|
"INSERT INTO dkim_keys (account_id, domain, selector, public_key, private_key_path, created_at) VALUES (?,?,?,?,?,NOW()) ON DUPLICATE KEY UPDATE selector=VALUES(selector), public_key=VALUES(public_key), private_key_path=VALUES(private_key_path)",
|
||||||
|
[$acctId, $domain, $selector, $pubKey, "{$keyDir}/{$selector}.private"]
|
||||||
|
);
|
||||||
|
|
||||||
|
// Add new TXT record, remove old mail._domainkey
|
||||||
|
DNSManager::addRecord($acctId, $domain, 'TXT', "{$selector}._domainkey", "v=DKIM1; k=rsa; p={$pubKey}", 300);
|
||||||
|
novacpx_log('info', "DKIM rotated for $domain, new selector: $selector");
|
||||||
|
return $selector;
|
||||||
|
}
|
||||||
|
|
||||||
public static function getDiskUsage(string $homeDir): int {
|
public static function getDiskUsage(string $homeDir): int {
|
||||||
$out = trim(shell_exec("du -sm " . escapeshellarg($homeDir) . " 2>/dev/null | awk '{print $1}'") ?: '0');
|
$out = trim(shell_exec("du -sm " . escapeshellarg($homeDir) . " 2>/dev/null | awk '{print $1}'") ?: '0');
|
||||||
return (int)$out;
|
return (int)$out;
|
||||||
|
|||||||
@@ -0,0 +1,4 @@
|
|||||||
|
Options -Indexes
|
||||||
|
RewriteEngine On
|
||||||
|
RewriteCond %{REQUEST_FILENAME} !-f
|
||||||
|
RewriteRule ^(.*)$ index.php [QSA,L]
|
||||||
+117
-29
@@ -172,38 +172,90 @@
|
|||||||
|
|
||||||
// ── Updates ────────────────────────────────────────────────────────────────
|
// ── Updates ────────────────────────────────────────────────────────────────
|
||||||
async function updates() {
|
async function updates() {
|
||||||
const [ver, check] = await Promise.all([
|
const [ver, ncpxCheck, osCheck] = await Promise.all([
|
||||||
Nova.api('system', 'version'),
|
Nova.api('system', 'version'),
|
||||||
Nova.api('system', 'check-update'),
|
Nova.api('system', 'check-novacpx-update'),
|
||||||
|
Nova.api('system', 'check-os-update'),
|
||||||
]);
|
]);
|
||||||
const v = ver?.data || {};
|
const v = ver?.data || {};
|
||||||
const upd = check?.data || {};
|
const ncpx = ncpxCheck?.data || {};
|
||||||
const count = upd.updates_available || 0;
|
const os = osCheck?.data || {};
|
||||||
|
const ncpxCount = ncpx.updates_available || 0;
|
||||||
|
const osCount = os.upgradable || 0;
|
||||||
|
|
||||||
return `
|
return `
|
||||||
<div class="card">
|
<div class="page-header mb-3">
|
||||||
<div class="card-header">
|
<h2 class="page-title">Updates</h2>
|
||||||
<span class="card-title">NovaCPX Updates</span>
|
<p class="text-muted text-sm">Manage NovaCPX panel updates and OS package upgrades.</p>
|
||||||
${count > 0 ? Nova.badge(count + ' update' + (count > 1 ? 's' : '') + ' available', 'yellow') : Nova.badge('Up to date', 'green')}
|
|
||||||
</div>
|
|
||||||
<div class="card-body">
|
|
||||||
<div class="grid-2 mb-3">
|
|
||||||
<div><p class="text-muted text-sm">Installed Version</p><p class="font-bold">${v.installed_version}</p></div>
|
|
||||||
<div><p class="text-muted text-sm">Git Commit</p><code>${v.git_commit || '—'}</code></div>
|
|
||||||
<div><p class="text-muted text-sm">Branch</p><code>${v.git_branch || 'main'}</code></div>
|
|
||||||
<div><p class="text-muted text-sm">Dirty Working Tree</p><p>${v.git_dirty ? Nova.badge('Yes','yellow') : Nova.badge('No','green')}</p></div>
|
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
${count > 0 ? `
|
<!-- NovaCPX Panel Updates -->
|
||||||
|
<div class="card mb-3">
|
||||||
|
<div class="card-header">
|
||||||
|
<span class="card-title">
|
||||||
|
<svg class="icon-sm mr-1"><use href="/assets/img/nova-icons.svg#ni-updates"/></svg>
|
||||||
|
NovaCPX Panel
|
||||||
|
</span>
|
||||||
|
${ncpxCount > 0 ? Nova.badge(ncpxCount + ' commit' + (ncpxCount > 1 ? 's' : '') + ' available', 'yellow') : Nova.badge('Up to date', 'green')}
|
||||||
|
<button class="btn btn-ghost btn-sm ml-auto" onclick="adminPage('updates')">
|
||||||
|
<svg class="icon-xs"><use href="/assets/img/nova-icons.svg#ni-search"/></svg> Refresh
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
<div class="card-body">
|
||||||
|
<div class="grid-4 mb-3">
|
||||||
|
<div><p class="text-muted text-sm">Installed</p><p class="font-bold">${v.installed_version || '—'}</p></div>
|
||||||
|
<div><p class="text-muted text-sm">Commit</p><code>${ncpx.current_commit || v.git_commit || '—'}</code></div>
|
||||||
|
<div><p class="text-muted text-sm">Branch</p><code>${ncpx.branch || 'main'}</code></div>
|
||||||
|
<div><p class="text-muted text-sm">PHP</p><code>${v.php_version || '—'}</code></div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
${ncpxCount > 0 ? `
|
||||||
<div class="card mb-2" style="background:var(--bg3)">
|
<div class="card mb-2" style="background:var(--bg3)">
|
||||||
<div class="card-header"><span class="card-title">Pending Commits</span></div>
|
<div class="card-header"><span class="card-title">Pending Commits</span></div>
|
||||||
<div class="card-body terminal">
|
<div class="card-body terminal" style="max-height:140px;overflow-y:auto">
|
||||||
${upd.commits?.map(c => `<div>${c}</div>`).join('') || 'None'}
|
${ncpx.commits?.map(c => `<div>${Nova.escHtml(c)}</div>`).join('') || 'None'}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<button class="btn btn-primary" onclick="applyUpdate()">Apply Update</button>
|
<p class="text-muted text-sm mb-2">PHP syntax is validated before deploy. If the panel goes down after update, it will automatically restore from backup.</p>
|
||||||
|
<button class="btn btn-primary" id="ncpx-update-btn" onclick="applyNovaCPXUpdate()">
|
||||||
|
<svg class="icon-xs mr-1"><use href="/assets/img/nova-icons.svg#ni-updates"/></svg>
|
||||||
|
Update NovaCPX
|
||||||
|
</button>
|
||||||
` : `<p class="text-muted">NovaCPX is up to date.</p>`}
|
` : `<p class="text-muted">NovaCPX is up to date.</p>`}
|
||||||
</div>
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- OS Updates -->
|
||||||
|
<div class="card">
|
||||||
|
<div class="card-header">
|
||||||
|
<span class="card-title">
|
||||||
|
<svg class="icon-sm mr-1"><use href="/assets/img/nova-icons.svg#ni-server"/></svg>
|
||||||
|
Operating System Packages
|
||||||
|
</span>
|
||||||
|
${os.security_updates > 0 ? Nova.badge(os.security_updates + ' security', 'red') : ''}
|
||||||
|
${osCount > 0 ? Nova.badge(osCount + ' upgradable', 'yellow') : Nova.badge('All current', 'green')}
|
||||||
|
</div>
|
||||||
|
<div class="card-body">
|
||||||
|
${osCount > 0 ? `
|
||||||
|
<div class="table-wrap mb-2" style="max-height:200px;overflow-y:auto">
|
||||||
|
<table>
|
||||||
|
<thead><tr><th>Package</th><th>From</th><th>To</th></tr></thead>
|
||||||
|
<tbody>
|
||||||
|
${os.packages?.map(p => `<tr>
|
||||||
|
<td><code>${Nova.escHtml(p.name)}</code></td>
|
||||||
|
<td class="text-muted text-sm">${Nova.escHtml(p.from || '(new)')}</td>
|
||||||
|
<td class="text-sm">${Nova.escHtml(p.to)}</td>
|
||||||
|
</tr>`).join('') || ''}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
<p class="text-muted text-sm mb-2">Services are automatically restarted if an upgrade stops them. The NovaCPX web root is backed up before upgrade and restored if panel ports go down.</p>
|
||||||
|
<button class="btn btn-warning" id="os-update-btn" onclick="applyOSUpdate()">
|
||||||
|
<svg class="icon-xs mr-1"><use href="/assets/img/nova-icons.svg#ni-server"/></svg>
|
||||||
|
Apply OS Upgrade
|
||||||
|
</button>
|
||||||
|
` : `<p class="text-muted">All OS packages are current.</p>`}
|
||||||
|
</div>
|
||||||
</div>`;
|
</div>`;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -868,18 +920,49 @@
|
|||||||
|
|
||||||
// ── Global action helpers ──────────────────────────────────────────────────
|
// ── Global action helpers ──────────────────────────────────────────────────
|
||||||
window.adminPage = (page) => Nova.loadPage(page, pages);
|
window.adminPage = (page) => Nova.loadPage(page, pages);
|
||||||
window.applyUpdate = async () => {
|
|
||||||
Nova.confirm('Apply all pending updates? The panel may restart.', async () => {
|
window.applyNovaCPXUpdate = async () => {
|
||||||
Nova.toast('Applying update…', 'info', 8000);
|
Nova.confirm('Apply NovaCPX update? PHP syntax is checked first, and a backup is taken automatically. The panel will self-restore if anything breaks.', async () => {
|
||||||
const res = await Nova.api('system', 'apply-update', { method: 'POST' });
|
const btn = document.getElementById('ncpx-update-btn');
|
||||||
|
if (btn) { btn.disabled = true; btn.textContent = 'Updating…'; }
|
||||||
|
Nova.toast('Pulling update from GitHub…', 'info', 12000);
|
||||||
|
const res = await Nova.api('system', 'apply-novacpx-update', { method: 'POST' });
|
||||||
if (res?.data?.updated) {
|
if (res?.data?.updated) {
|
||||||
Nova.toast(`Updated to ${res.data.to_commit}`, 'success');
|
Nova.toast(`Updated to ${res.data.to_commit}`, 'success', 6000);
|
||||||
Nova.loadPage('updates', pages);
|
setTimeout(() => Nova.loadPage('updates', pages), 2000);
|
||||||
|
} else if (res?.error) {
|
||||||
|
Nova.toast(res.error, 'error', 8000);
|
||||||
|
if (btn) { btn.disabled = false; btn.textContent = 'Update NovaCPX'; }
|
||||||
} else {
|
} else {
|
||||||
Nova.toast(res?.data?.pull_output || 'Already up to date', 'info');
|
Nova.toast('Already up to date.', 'info');
|
||||||
|
if (btn) { btn.disabled = false; btn.textContent = 'Update NovaCPX'; }
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
|
window.applyOSUpdate = async () => {
|
||||||
|
Nova.confirm('Apply OS package upgrades? Services will be automatically restarted if needed. The NovaCPX panel will self-restore from backup if any ports go down.', async () => {
|
||||||
|
const btn = document.getElementById('os-update-btn');
|
||||||
|
if (btn) { btn.disabled = true; btn.textContent = 'Upgrading…'; }
|
||||||
|
Nova.toast('Running apt-get upgrade — this may take a few minutes…', 'info', 20000);
|
||||||
|
const res = await Nova.api('system', 'apply-os-update', { method: 'POST', timeout: 120000 });
|
||||||
|
if (res?.data) {
|
||||||
|
const d = res.data;
|
||||||
|
const healed = Object.entries(d.services_healed || {}).map(([s,r]) => `${s}: ${r}`).join(', ');
|
||||||
|
let msg = 'OS upgrade complete.';
|
||||||
|
if (healed) msg += ` Auto-healed: ${healed}.`;
|
||||||
|
if (!d.panel_ports_ok) msg += ' ⚠ Panel ports were down — auto-restored from backup.';
|
||||||
|
Nova.toast(msg, d.panel_ports_ok ? 'success' : 'warning', 10000);
|
||||||
|
Nova.loadPage('updates', pages);
|
||||||
|
} else {
|
||||||
|
Nova.toast(res?.error || 'Upgrade failed', 'error', 8000);
|
||||||
|
if (btn) { btn.disabled = false; btn.textContent = 'Apply OS Upgrade'; }
|
||||||
|
}
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
// keep old alias for any lingering references
|
||||||
|
window.applyUpdate = window.applyNovaCPXUpdate;
|
||||||
window.adminServiceAction = async (svc, cmd) => {
|
window.adminServiceAction = async (svc, cmd) => {
|
||||||
const res = await Nova.api('system', 'service', { method: 'POST', body: { service: svc, command: cmd } });
|
const res = await Nova.api('system', 'service', { method: 'POST', body: { service: svc, command: cmd } });
|
||||||
Nova.toast(`${svc}: ${cmd} → ${res?.success ? 'OK' : res?.message}`, res?.success ? 'success' : 'error');
|
Nova.toast(`${svc}: ${cmd} → ${res?.success ? 'OK' : res?.message}`, res?.success ? 'success' : 'error');
|
||||||
@@ -891,9 +974,14 @@
|
|||||||
|
|
||||||
// ── Check for updates badge ────────────────────────────────────────────────
|
// ── Check for updates badge ────────────────────────────────────────────────
|
||||||
async function checkUpdates() {
|
async function checkUpdates() {
|
||||||
const res = await Nova.api('system', 'check-update');
|
const [ncpx, os] = await Promise.all([
|
||||||
const n = res?.data?.updates_available || 0;
|
Nova.api('system', 'check-novacpx-update'),
|
||||||
|
Nova.api('system', 'check-os-update'),
|
||||||
|
]);
|
||||||
|
const ncpxN = ncpx?.data?.updates_available || 0;
|
||||||
|
const osN = os?.data?.upgradable || 0;
|
||||||
|
const total = ncpxN + osN;
|
||||||
const badge = document.getElementById('update-badge');
|
const badge = document.getElementById('update-badge');
|
||||||
if (badge && n > 0) { badge.textContent = n; badge.style.display = ''; }
|
if (badge && total > 0) { badge.textContent = total; badge.style.display = ''; }
|
||||||
}
|
}
|
||||||
})();
|
})();
|
||||||
|
|||||||
Reference in New Issue
Block a user