Phase 4: Vision Protocol — screenshot + Claude vision

- reactor.py: v4.0.0; adds screenshot, vision, sysinfo handlers;
  _dispatch_agent_command() shared helper; FastAPI /screenshots endpoints
- jarvis-agent.py: v3.0; screenshot command handler (scrot/import/fbcat/
  ImageMagick render fallback); sysinfo command returns structured snapshot;
  detect_capabilities() advertises screenshot + sysinfo caps
- chat.php: Tier 0.9c detects screenshot (show screen on X, screenshot X)
  and sysinfo (check status of X) voice/text commands
- arc.php: screenshots, screenshot_get, screenshot_delete actions
- index.html: VISION PROTOCOL lightbox overlay; SCREENSHOT + SYSINFO
  buttons on each online agent card; keyboard Escape to close
- admin/index.php: VISION PROTOCOL tab under ARC REACTOR nav; gallery view
  with image thumbnails + analysis; take screenshot modal; purge action
This commit is contained in:
2026-06-11 04:42:21 +00:00
parent 068aff27b4
commit 56c9e2d914
5 changed files with 576 additions and 3 deletions
+205 -1
View File
@@ -23,7 +23,7 @@ from pathlib import Path
CONFIG_PATH = "/etc/jarvis-agent/config.json" CONFIG_PATH = "/etc/jarvis-agent/config.json"
STATE_PATH = "/var/lib/jarvis-agent/state.json" STATE_PATH = "/var/lib/jarvis-agent/state.json"
AGENT_VERSION = "2.3" # bumped on each release AGENT_VERSION = "3.0" # Phase 4: screenshot + sysinfo commands
# ── Config helpers ──────────────────────────────────────────────────────────── # ── Config helpers ────────────────────────────────────────────────────────────
@@ -119,6 +119,12 @@ def detect_capabilities(cfg: dict) -> list:
# Check for Home Assistant # Check for Home Assistant
if os.path.exists("/etc/homeassistant") or os.path.exists("/config/configuration.yaml"): if os.path.exists("/etc/homeassistant") or os.path.exists("/config/configuration.yaml"):
caps.append("homeassistant") caps.append("homeassistant")
# Phase 4: screenshot capability
import shutil as _shutil
if (_shutil.which("scrot") or _shutil.which("import") or
_shutil.which("fbcat") or _shutil.which("convert")):
caps.append("screenshot")
caps.append("sysinfo")
return caps return caps
def register(cfg: dict, state: dict) -> str: def register(cfg: dict, state: dict) -> str:
@@ -298,6 +304,198 @@ def collect_proxmox_metrics(cfg: dict) -> dict | None:
except Exception as e: except Exception as e:
return {"error": str(e)} return {"error": str(e)}
# ── Screenshot / Vision helpers ───────────────────────────────────────────────
def _take_screenshot(cmd_data: dict) -> dict:
"""
Attempts to capture a screenshot using available tools.
For headless servers, falls back to a rich text system snapshot.
Returns base64-encoded PNG and metadata.
"""
import base64, tempfile, shutil
tmp = tempfile.mktemp(suffix=".png")
width = height = 0
method = "unknown"
# 1. Try scrot (X11 desktop)
if shutil.which("scrot") and os.environ.get("DISPLAY"):
try:
r = subprocess.run(["scrot", "-z", tmp], capture_output=True, timeout=10)
if r.returncode == 0 and os.path.exists(tmp):
method = "scrot"
except Exception:
pass
# 2. Try import (ImageMagick X11)
if method == "unknown" and shutil.which("import") and os.environ.get("DISPLAY"):
try:
r = subprocess.run(["import", "-window", "root", tmp], capture_output=True, timeout=10)
if r.returncode == 0 and os.path.exists(tmp):
method = "import"
except Exception:
pass
# 3. Try fbcat (Linux framebuffer — headless VMs with framebuffer)
if method == "unknown" and shutil.which("fbcat") and os.path.exists("/dev/fb0"):
try:
ppm = tempfile.mktemp(suffix=".ppm")
r = subprocess.run(["fbcat", "-s", "/dev/fb0"], stdout=open(ppm, "wb"),
stderr=subprocess.PIPE, timeout=10)
if r.returncode == 0 and shutil.which("convert"):
subprocess.run(["convert", ppm, tmp], capture_output=True, timeout=10)
os.unlink(ppm)
if os.path.exists(tmp):
method = "framebuffer"
except Exception:
pass
# 4. Headless fallback: build a PNG system dashboard from text stats
if method == "unknown":
try:
result = _render_sysinfo_png(tmp)
if result:
method = "sysinfo_render"
except Exception:
pass
if method == "unknown" or not os.path.exists(tmp):
# Last resort: return text snapshot only
snap = _sysinfo_snapshot()
snap["screenshot_available"] = False
snap["method"] = "text_only"
return snap
# Read image
try:
with open(tmp, "rb") as f:
raw = f.read()
b64 = base64.b64encode(raw).decode()
fsize = len(raw)
os.unlink(tmp)
# Try to get dimensions via file command
try:
r = subprocess.run(["identify", "-format", "%wx%h", tmp],
capture_output=True, text=True, timeout=5)
if "x" in r.stdout:
w, h = r.stdout.strip().split("x", 1)
width, height = int(w), int(h)
except Exception:
pass
return {
"success": True,
"method": method,
"image_b64": b64,
"image_mime": "image/png",
"file_size": fsize,
"width": width,
"height": height,
"hostname": socket.gethostname(),
}
except Exception as e:
return {"success": False, "error": str(e), "method": method}
def _render_sysinfo_png(out_path: str) -> bool:
"""Render a system info text snapshot as a PNG using ansi2image or ImageMagick."""
import shutil
snap = _build_sysinfo_text()
# Try convert (ImageMagick) to render text → PNG
if shutil.which("convert"):
try:
r = subprocess.run([
"convert",
"-size", "900x600", "xc:#0a0f14",
"-font", "Courier-New",
"-pointsize", "13",
"-fill", "#00d4ff",
"-annotate", "+20+30", snap[:3000],
out_path,
], capture_output=True, timeout=15)
return r.returncode == 0 and os.path.exists(out_path)
except Exception:
pass
return False
def _build_sysinfo_text() -> str:
"""Build a rich text system snapshot for headless machines."""
lines = [f"JARVIS FIELD STATION — {socket.gethostname()}",
f"Timestamp: {datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S UTC')}",
"" * 60]
try:
# CPU / mem / disk
with open("/proc/loadavg") as f:
load = f.read().split()[:3]
lines.append(f"Load avg: {' '.join(load)}")
except Exception:
pass
try:
with open("/proc/meminfo") as f:
minfo = {l.split(":")[0].strip(): int(l.split()[1]) for l in f if ":" in l}
total = minfo.get("MemTotal", 0)
avail = minfo.get("MemAvailable", 0)
used = total - avail
lines.append(f"Memory: {used//1024}MB used / {total//1024}MB total")
except Exception:
pass
try:
r = subprocess.run(["df", "-h", "/"], capture_output=True, text=True, timeout=5)
lines.append("Disk:\n" + r.stdout.strip())
except Exception:
pass
try:
r = subprocess.run(["ps", "aux", "--sort=-%cpu"], capture_output=True, text=True, timeout=5)
lines.append("Top processes:\n" + "\n".join(r.stdout.splitlines()[1:8]))
except Exception:
pass
try:
r = subprocess.run(["ss", "-tlnp"], capture_output=True, text=True, timeout=5)
lines.append("Listening ports:\n" + r.stdout.strip()[:500])
except Exception:
pass
return "\n".join(lines)
def _sysinfo_snapshot() -> dict:
"""Return structured system snapshot (no image) for text-based analysis."""
data = {"success": True, "hostname": socket.gethostname(),
"snapshot_type": "text", "screenshot_available": False}
try:
with open("/proc/loadavg") as f:
parts = f.read().split()
data["load_1m"], data["load_5m"], data["load_15m"] = parts[0], parts[1], parts[2]
except Exception:
pass
try:
with open("/proc/meminfo") as f:
m = {l.split(":")[0].strip(): int(l.split()[1])
for l in f if ":" in l and len(l.split()) >= 2}
data["mem_total_mb"] = m.get("MemTotal", 0) // 1024
data["mem_avail_mb"] = m.get("MemAvailable", 0) // 1024
data["mem_used_mb"] = data["mem_total_mb"] - data["mem_avail_mb"]
except Exception:
pass
try:
r = subprocess.run(["df", "-h", "/"], capture_output=True, text=True, timeout=5)
data["disk"] = r.stdout.splitlines()[1] if r.stdout else ""
except Exception:
pass
try:
r = subprocess.run(["ps", "aux", "--sort=-%cpu"], capture_output=True, text=True, timeout=5)
data["top_procs"] = r.stdout.splitlines()[1:8]
except Exception:
pass
try:
r = subprocess.run(["ss", "-tlnp"], capture_output=True, text=True, timeout=5)
data["listening_ports"] = r.stdout.strip()[:800]
except Exception:
pass
return data
# ── Command execution ───────────────────────────────────────────────────────── # ── Command execution ─────────────────────────────────────────────────────────
def execute_command(cmd: dict) -> dict: def execute_command(cmd: dict) -> dict:
@@ -338,6 +536,12 @@ def execute_command(cmd: dict) -> dict:
r = subprocess.run(cmd_str, shell=True, capture_output=True, text=True, timeout=30) r = subprocess.run(cmd_str, shell=True, capture_output=True, text=True, timeout=30)
return {"success": True, "stdout": r.stdout[:2000], "stderr": r.stderr[:500]} return {"success": True, "stdout": r.stdout[:2000], "stderr": r.stderr[:500]}
elif cmd_type == "screenshot":
return _take_screenshot(cmd_data)
elif cmd_type == "sysinfo":
return _sysinfo_snapshot()
else: else:
return {"success": False, "error": f"Unknown command type: {cmd_type}"} return {"success": False, "error": f"Unknown command type: {cmd_type}"}
+25
View File
@@ -154,6 +154,31 @@ switch ($action) {
echo json_encode(['ok' => true, 'id' => $id, 'action_taken' => $actionTaken]); echo json_encode(['ok' => true, 'id' => $id, 'action_taken' => $actionTaken]);
break; break;
// GET /api/arc?action=screenshots&limit=20&agent=hostname
case 'screenshots':
$limit = min((int)($_GET['limit'] ?? 20), 100);
$agent = $_GET['agent'] ?? '';
$url = 'http://127.0.0.1:7474/screenshots?' . http_build_query(array_filter(['limit' => $limit, 'agent' => $agent]));
$ch = curl_init($url);
curl_setopt_array($ch, [CURLOPT_RETURNTRANSFER => true, CURLOPT_TIMEOUT => 5]);
$raw = curl_exec($ch); curl_close($ch);
echo $raw ?: '[]';
break;
// GET /api/arc?action=screenshot_get&id=123
case 'screenshot_get':
$id = (int)($_GET['id'] ?? 0);
if (!$id) { http_response_code(400); echo json_encode(['error' => 'Missing id']); break; }
echo json_encode(arc_request('GET', "/screenshots/{$id}"));
break;
// DELETE /api/arc?action=screenshot_delete&id=123
case 'screenshot_delete':
$id = (int)($_GET['id'] ?? 0);
if (!$id) { http_response_code(400); echo json_encode(['error' => 'Missing id']); break; }
echo json_encode(arc_request('DELETE', "/screenshots/{$id}"));
break;
default: default:
http_response_code(404); http_response_code(404);
echo json_encode(['error' => "Unknown arc action: {$action}"]); echo json_encode(['error' => "Unknown arc action: {$action}"]);
+34 -2
View File
@@ -1063,7 +1063,7 @@ if (!$reply && preg_match('/\b(news|headlines|latest|what.?s happening|current e
} }
} }
// ── Tier 0.9: Intel Protocol — research, tool_loop, gmail_triage, remote_exec ─ // ── Tier 0.9: Arc Protocols — research, triage, remote_exec, screenshot, sysinfo
$arcJobId = null; $arcJobId = null;
// Helper: submit job to Arc Reactor // Helper: submit job to Arc Reactor
@@ -1158,7 +1158,39 @@ if (!$reply) {
} }
} }
// ── Tier 0.9c: Intel Protocol — research & tool_loop detection ──────────── // ── Tier 0.9c: Vision Protocol — screenshot, sysinfo, vision detection ───────
if (!$reply) {
// Screenshot patterns
$screenshotMatch = null;
if (preg_match('/^(?:jarvis[,\s]+)?(?:show\s+(?:me\s+)?(?:the\s+)?screen\s+(?:on|of|from)|screenshot\s+(?:of\s+)?|grab\s+(?:a\s+)?(?:screenshot|screen\s+cap)\s+(?:of\s+|from\s+)?|what(?:\'s|\s+is)\s+(?:on|showing\s+on)\s+(?:the\s+)?screen\s+(?:on|of))\s+(.+)/i', $message, $mm)) {
$screenshotMatch = trim($mm[1]);
$arcRes = arcSubmitJob('screenshot', ['agent' => $screenshotMatch, 'analyze' => true], $sessionId);
if (isset($arcRes['job_id'])) {
$arcJobId = $arcRes['job_id'];
$reply = "◈ VISION PROTOCOL ACTIVATED — Capturing screen on **{$screenshotMatch}** (Job #{$arcJobId}). I'll analyze what I see and report back, {$userAddr}.";
$source = 'arc:screenshot';
} else {
$reply = "Vision Protocol is offline, {$userAddr}. Arc Reactor may be unavailable.";
$source = 'arc:offline';
}
}
// Sysinfo patterns
elseif (preg_match('/^(?:jarvis[,\s]+)?(?:(?:what(?:\'s|\s+is)\s+(?:the\s+)?status\s+of|check\s+(?:the\s+)?(?:health|status)\s+of|how\s+is)\s+(.+)|system\s+(?:info|status|snapshot)\s+(?:on|from|for)\s+(.+))/i', $message, $mm)) {
$agentName = trim($mm[1] ?? $mm[2] ?? '');
// Only fire for explicit agent references (has a name), not generic questions
if ($agentName && strlen($agentName) > 2 &&
!preg_match('/\b(?:weather|today|things|everything|jarvis|yourself)\b/i', $agentName)) {
$arcRes = arcSubmitJob('sysinfo', ['agent' => $agentName, 'analyze' => true], $sessionId);
if (isset($arcRes['job_id'])) {
$arcJobId = $arcRes['job_id'];
$reply = "◈ FIELD PROTOCOL — Running system health check on **{$agentName}** (Job #{$arcJobId}). Snapshot incoming, {$userAddr}.";
$source = 'arc:sysinfo';
}
}
}
}
// ── Tier 0.9e: Intel Protocol — research & tool_loop detection ────────────
$intelPatterns = [ $intelPatterns = [
'/^(?:jarvis[,\s]+)?(?:research|investigate|deep[- ]dive|deep dive)\s+(.+)/i' => 'research', '/^(?:jarvis[,\s]+)?(?:research|investigate|deep[- ]dive|deep dive)\s+(.+)/i' => 'research',
'/^(?:jarvis[,\s]+)?(?:look\s+(?:up|into)|find\s+out\s+(?:about)?)\s+(.+)/i' => 'research', '/^(?:jarvis[,\s]+)?(?:look\s+(?:up|into)|find\s+out\s+(?:about)?)\s+(.+)/i' => 'research',
+179
View File
@@ -550,6 +550,48 @@ if ($action) {
$raw = curl_exec($ch); curl_close($ch); $raw = curl_exec($ch); curl_close($ch);
j(json_decode($raw, true) ?: ['error' => 'Arc Reactor unreachable']); j(json_decode($raw, true) ?: ['error' => 'Arc Reactor unreachable']);
// ── VISION PROTOCOL ──────────────────────────────────────────────────
case 'vision_list':
$limit = min((int)($_GET['limit'] ?? 30), 100);
$agent = $_GET['agent'] ?? '';
$url = 'http://127.0.0.1:7474/screenshots?' . http_build_query(array_filter(['limit'=>$limit,'agent'=>$agent]));
$ch = curl_init($url);
curl_setopt_array($ch, [CURLOPT_RETURNTRANSFER=>true, CURLOPT_TIMEOUT=>5]);
$raw = curl_exec($ch); curl_close($ch);
j(json_decode($raw, true) ?: []);
case 'vision_get':
$id = (int)($_GET['id'] ?? 0); if (!$id) bad('Missing id');
$ch = curl_init('http://127.0.0.1:7474/screenshots/' . $id);
curl_setopt_array($ch, [CURLOPT_RETURNTRANSFER=>true, CURLOPT_TIMEOUT=>5]);
$raw = curl_exec($ch); curl_close($ch);
j(json_decode($raw, true) ?: ['error'=>'not found']);
case 'vision_delete':
$id = (int)($_GET['id'] ?? 0); if (!$id) bad('Missing id');
$ch = curl_init('http://127.0.0.1:7474/screenshots/' . $id);
curl_setopt_array($ch, [CURLOPT_RETURNTRANSFER=>true, CURLOPT_TIMEOUT=>5, CURLOPT_CUSTOMREQUEST=>'DELETE']);
$raw = curl_exec($ch); curl_close($ch);
j(json_decode($raw, true) ?: ['ok'=>true]);
case 'vision_screenshot':
$agent = trim($_GET['agent'] ?? ''); if (!$agent) bad('Missing agent');
$analyze = ($_GET['analyze'] ?? '1') !== '0';
$ch = curl_init('http://127.0.0.1:7474/job');
curl_setopt_array($ch, [
CURLOPT_RETURNTRANSFER => true, CURLOPT_TIMEOUT => 5, CURLOPT_POST => true,
CURLOPT_POSTFIELDS => json_encode(['type'=>'screenshot','payload'=>['agent'=>$agent,'analyze'=>$analyze],'priority'=>8,'created_by'=>'admin']),
CURLOPT_HTTPHEADER => ['Content-Type: application/json'],
]);
$raw = curl_exec($ch); curl_close($ch);
j(json_decode($raw, true) ?: ['error'=>'Arc Reactor unreachable']);
case 'vision_purge':
$ch = curl_init('http://127.0.0.1:7474/screenshots/purge');
curl_setopt_array($ch, [CURLOPT_RETURNTRANSFER=>true, CURLOPT_TIMEOUT=>5, CURLOPT_CUSTOMREQUEST=>'DELETE']);
$raw = curl_exec($ch); curl_close($ch);
j(json_decode($raw, true) ?: ['ok'=>true]);
case 'users_list': case 'users_list':
j(JarvisDB::query('SELECT id,username,display_name,last_seen,created_at FROM users ORDER BY username')); j(JarvisDB::query('SELECT id,username,display_name,last_seen,created_at FROM users ORDER BY username'));
@@ -789,6 +831,7 @@ select.filter-sel:focus{border-color:var(--cyan)}
<div class="nav-item" data-tab="calendar" onclick="nav(this)">🗓 CALENDAR SYNC</div> <div class="nav-item" data-tab="calendar" onclick="nav(this)">🗓 CALENDAR SYNC</div>
<div class="nav-section">ARC REACTOR</div> <div class="nav-section">ARC REACTOR</div>
<div class="nav-item" data-tab="arc" onclick="nav(this)"> ARC REACTOR</div> <div class="nav-item" data-tab="arc" onclick="nav(this)"> ARC REACTOR</div>
<div class="nav-item" data-tab="vision" onclick="nav(this)"> VISION PROTOCOL</div>
<div class="nav-section">INFO</div> <div class="nav-section">INFO</div>
<div class="nav-item" data-tab="sites" onclick="nav(this)">SITES</div> <div class="nav-item" data-tab="sites" onclick="nav(this)">SITES</div>
<div class="nav-item" data-tab="users" onclick="nav(this)">USERS</div> <div class="nav-item" data-tab="users" onclick="nav(this)">USERS</div>
@@ -1085,6 +1128,25 @@ select.filter-sel:focus{border-color:var(--cyan)}
<div class="tbl-wrap" id="arc-jobs-tbl"><div class="loading">INITIALIZING...</div></div> <div class="tbl-wrap" id="arc-jobs-tbl"><div class="loading">INITIALIZING...</div></div>
</div> </div>
<!-- VISION PROTOCOL -->
<div class="tab" id="tab-vision">
<div class="page-title"> VISION PROTOCOL FIELD SCREENSHOTS</div>
<div style="display:flex;gap:10px;margin-bottom:16px;align-items:center;flex-wrap:wrap">
<button class="btn btn-sm btn-green" onclick="visionRunScreenshot()"> TAKE SCREENSHOT</button>
<button class="btn btn-sm" onclick="loadVision()"> REFRESH</button>
<select id="vision-agent-filter" class="inp" style="width:auto;padding:4px 8px;font-size:0.65rem" onchange="loadVision()">
<option value="">ALL AGENTS</option>
</select>
<button class="btn btn-sm" onclick="visionPurge()" style="margin-left:auto;opacity:0.7">PURGE OLD</button>
<div id="vision-count" style="font-family:var(--mono);font-size:0.65rem;color:var(--dim)"></div>
</div>
<div id="vision-gallery" style="display:grid;grid-template-columns:repeat(auto-fill,minmax(280px,1fr));gap:14px">
<div class="loading">LOADING SCREENSHOTS...</div>
</div>
</div>
<!-- GMAIL TRIAGE --> <!-- GMAIL TRIAGE -->
<div class="tab" id="tab-triage"> <div class="tab" id="tab-triage">
<div class="page-title"> COMMS PROTOCOL GMAIL TRIAGE</div> <div class="page-title"> COMMS PROTOCOL GMAIL TRIAGE</div>
@@ -1219,6 +1281,7 @@ function loadTab(tab) {
users: loadUsers, users: loadUsers,
email: ()=>{ loadEmailInbox(); loadEmailActionItems(); }, email: ()=>{ loadEmailInbox(); loadEmailActionItems(); },
triage: loadTriage, triage: loadTriage,
vision: loadVision,
tasks: loadTasks, tasks: loadTasks,
appointments: loadAppts, appointments: loadAppts,
calendar: loadCalFeeds, calendar: loadCalFeeds,
@@ -2110,6 +2173,122 @@ function emailDismiss(id) {
apiPost('email_dismiss',{id},()=>{ toast('Dismissed','ok'); loadEmailActionItems(); }); apiPost('email_dismiss',{id},()=>{ toast('Dismissed','ok'); loadEmailActionItems(); });
} }
// ── VISION PROTOCOL ──────────────────────────────────────────────────────────
let _visionAgents = [];
async function loadVision() {
const gallery = document.getElementById('vision-gallery');
if (!gallery) return;
gallery.innerHTML = '<div class="loading">LOADING...</div>';
const filter = document.getElementById('vision-agent-filter')?.value || '';
const shots = await api('vision_list', {limit: 30, agent: filter});
// Populate agent filter dropdown
const select = document.getElementById('vision-agent-filter');
if (select && Array.isArray(shots) && shots.length) {
const agents = [...new Set(shots.map(s => s.hostname).filter(Boolean))];
_visionAgents = agents;
const current = select.value;
select.innerHTML = '<option value="">ALL AGENTS</option>' +
agents.map(a => `<option value="${esc(a)}"${a===current?' selected':''}>${esc(a)}</option>`).join('');
}
document.getElementById('vision-count').textContent = Array.isArray(shots) ? shots.length + ' SCREENSHOTS' : '';
if (!Array.isArray(shots) || !shots.length) {
gallery.innerHTML = '<div class="loading">No screenshots yet. Click "TAKE SCREENSHOT" to capture a field station.</div>';
return;
}
gallery.innerHTML = shots.map(s => {
const ts = ts(s.created_at);
const has = s.file_size > 0;
const meth = (s.method || 'unknown').toUpperCase();
const dim = s.width && s.height ? `${s.width}×${s.height}` : '';
const analysis = (s.vision_analysis || '').substring(0, 180);
return `<div style="background:rgba(0,212,255,0.03);border:1px solid var(--border);border-radius:4px;overflow:hidden">
<div style="background:rgba(0,212,255,0.06);padding:8px 10px;display:flex;align-items:center;gap:8px">
<span style="font-family:var(--mono);font-size:0.65rem;flex:1;white-space:nowrap;overflow:hidden;text-overflow:ellipsis">${esc(s.hostname||'unknown')}</span>
<span style="font-family:var(--mono);font-size:0.55rem;color:var(--dim)">${meth}</span>
<button class="btn btn-xs" onclick="visionViewScreenshot(${s.id})" style="border-color:var(--cyan);color:var(--cyan)">VIEW</button>
<button class="btn btn-xs" onclick="visionDeleteShot(${s.id})"></button>
</div>
<div style="padding:8px 10px">
${has ? `<div style="background:#060a0e;border:1px solid var(--border);border-radius:3px;padding:5px;margin-bottom:8px;cursor:pointer;font-family:var(--mono);font-size:0.6rem;color:var(--text-dim);text-align:center" onclick="visionViewScreenshot(${s.id})">
${dim ? dim + ' · ' : ''}${Math.round((s.file_size||0)/1024)}KB IMAGE
</div>` : '<div style="font-size:0.6rem;color:var(--dim);margin-bottom:6px;font-family:var(--mono)">TEXT SNAPSHOT ONLY</div>'}
${analysis ? `<div style="font-size:0.62rem;line-height:1.5;color:var(--text-dim)">${esc(analysis)}${s.vision_analysis?.length>180?'…':''}</div>` : ''}
<div style="font-family:var(--mono);font-size:0.55rem;color:var(--border2);margin-top:6px">${ts}</div>
</div>
</div>`;
}).join('');
}
async function visionViewScreenshot(id) {
const d = await api('vision_get', {id});
if (d.error) { toast('Error: ' + d.error, 'err'); return; }
const imgHtml = d.image_b64
? `<img src="data:image/png;base64,${d.image_b64}" style="max-width:100%;border:1px solid var(--border);border-radius:3px;margin-bottom:12px">`
: '<div style="color:var(--dim);font-family:var(--mono);font-size:0.65rem;padding:12px 0">No image data — text snapshot only</div>';
openModal('◈ VISION — ' + esc(d.hostname||''), `
<div style="font-size:0.6rem;color:var(--dim);margin-bottom:10px;font-family:var(--mono)">
METHOD: ${esc(d.method||'')} · ${d.width&&d.height?d.width+'×'+d.height+' · ':''} ${Math.round((d.file_size||0)/1024)}KB · ${ts(d.created_at)}
</div>
${imgHtml}
${d.vision_analysis ? `<div style="font-size:0.55rem;letter-spacing:2px;color:var(--dim);margin-bottom:6px">VISION ANALYSIS</div>
<pre style="white-space:pre-wrap;font-size:0.65rem;line-height:1.6;color:var(--text);background:rgba(0,212,255,0.04);border:1px solid var(--border);padding:10px;border-radius:3px;max-height:300px;overflow-y:auto">${esc(d.vision_analysis)}</pre>` : ''}
`, null, null);
document.getElementById('modalSave').style.display = 'none';
}
async function visionRunScreenshot() {
// Pick agent from dropdown or prompt
const agents = _visionAgents;
if (!agents.length) {
toast('No agents online — check AGENTS tab', 'err'); return;
}
// Build select for agent
openModal('◈ TAKE SCREENSHOT', `
<div style="margin-bottom:14px">
<label style="font-size:0.65rem;color:var(--dim);display:block;margin-bottom:6px">SELECT FIELD AGENT</label>
<select id="vision-agent-sel" class="inp" style="width:100%">
${agents.map(a => `<option value="${esc(a)}">${esc(a)}</option>`).join('')}
</select>
</div>
<div>
<label style="font-size:0.65rem;color:var(--dim);display:flex;align-items:center;gap:8px;cursor:pointer">
<input type="checkbox" id="vision-analyze-chk" checked> Run Claude vision analysis
</label>
</div>
`, async () => {
const agent = document.getElementById('vision-agent-sel')?.value || '';
const analyze = document.getElementById('vision-analyze-chk')?.checked ? '1' : '0';
const d = await api('vision_screenshot', {agent, analyze});
if (d.job_id) {
toast('Screenshot job started — Job #' + d.job_id, 'ok');
setTimeout(() => loadVision(), 5000);
} else {
toast('Failed: ' + (d.error || 'Arc offline'), 'err');
}
closeModal();
}, 'CAPTURE');
}
async function visionDeleteShot(id) {
await api('vision_delete', {id});
toast('Deleted', 'ok');
loadVision();
}
async function visionPurge() {
await api('vision_purge');
toast('Old screenshots purged', 'ok');
loadVision();
}
// ── GMAIL TRIAGE ───────────────────────────────────────────────────────────── // ── GMAIL TRIAGE ─────────────────────────────────────────────────────────────
const _TRIAGE_COLORS = {urgent:'var(--red)',action:'var(--orange)',reply:'var(--cyan)',meeting:'#a78bfa',info:'var(--text-dim)',promo:'rgba(255,255,255,0.25)',spam:'rgba(255,255,255,0.15)'}; const _TRIAGE_COLORS = {urgent:'var(--red)',action:'var(--orange)',reply:'var(--cyan)',meeting:'#a78bfa',info:'var(--text-dim)',promo:'rgba(255,255,255,0.25)',spam:'rgba(255,255,255,0.15)'};
+133
View File
@@ -875,6 +875,15 @@ body::after{
transition:filter 1.2s ease; transition:filter 1.2s ease;
} }
#app.sleeping #sleepOverlay{display:flex} #app.sleeping #sleepOverlay{display:flex}
/* ── VISION PROTOCOL — screenshot lightbox ───────────────────────── */
#vision-lightbox{display:none;position:fixed;inset:0;background:rgba(0,0,0,0.88);z-index:9999;flex-direction:column;align-items:center;justify-content:flex-start;padding:20px;overflow-y:auto}
#vision-lightbox.open{display:flex}
#vision-lb-header{width:100%;max-width:960px;display:flex;align-items:center;gap:10px;margin-bottom:12px}
#vision-lb-title{font-family:var(--font-display);font-size:0.65rem;letter-spacing:2px;color:var(--cyan);flex:1}
#vision-lb-close{background:none;border:1px solid var(--panel-border);color:var(--text-dim);padding:3px 10px;border-radius:3px;cursor:pointer;font-family:var(--font-display);font-size:0.6rem}
#vision-lb-img{max-width:960px;width:100%;border:1px solid var(--panel-border);border-radius:4px;margin-bottom:12px}
#vision-lb-analysis{max-width:960px;width:100%;background:rgba(0,212,255,0.04);border:1px solid var(--panel-border);border-radius:4px;padding:14px 16px;font-size:0.65rem;line-height:1.7;color:var(--text);white-space:pre-wrap}
#vision-lb-spinner{color:var(--cyan);font-family:var(--font-display);font-size:0.65rem;letter-spacing:2px;animation:pulse 1.5s ease-in-out infinite;margin:30px auto}
/* ── COMMS PROTOCOL — email triage cards ─────────────────────────── */ /* ── COMMS PROTOCOL — email triage cards ─────────────────────────── */
.comms-card{background:rgba(0,212,255,0.04);border:1px solid var(--panel-border);border-radius:var(--r);margin-bottom:7px;overflow:hidden} .comms-card{background:rgba(0,212,255,0.04);border:1px solid var(--panel-border);border-radius:var(--r);margin-bottom:7px;overflow:hidden}
.comms-card-head{display:flex;align-items:center;gap:7px;padding:7px 10px;cursor:pointer;user-select:none} .comms-card-head{display:flex;align-items:center;gap:7px;padding:7px 10px;cursor:pointer;user-select:none}
@@ -3879,6 +3888,10 @@ function renderAgentsTab(agents, metrics) {
<div style="font-size:0.58rem;color:var(--text-dim)">UP: ${uptime} · SEEN: ${since}</div> <div style="font-size:0.58rem;color:var(--text-dim)">UP: ${uptime} · SEEN: ${since}</div>
${svcs ? `<div style="font-size:0.58rem">${svcs}</div>` : ''} ${svcs ? `<div style="font-size:0.58rem">${svcs}</div>` : ''}
</div> </div>
${alive ? `<div style="display:flex;gap:5px;margin-top:6px">
<button onclick="event.stopPropagation();agentScreenshot('${ag.hostname}')" style="flex:1;background:rgba(0,212,255,0.06);border:1px solid var(--panel-border);border-radius:3px;padding:3px 6px;color:var(--cyan);font-family:var(--font-display);font-size:0.48rem;letter-spacing:1px;cursor:pointer">◈ SCREENSHOT</button>
<button onclick="event.stopPropagation();agentSysinfo('${ag.hostname}')" style="flex:1;background:rgba(0,212,255,0.06);border:1px solid var(--panel-border);border-radius:3px;padding:3px 6px;color:var(--text-dim);font-family:var(--font-display);font-size:0.48rem;letter-spacing:1px;cursor:pointer">⚡ SYSINFO</button>
</div>` : ''}
</div>`; </div>`;
}).join(''); }).join('');
} }
@@ -4056,6 +4069,126 @@ async function saveSite(id) {
} }
} }
// ── VISION PROTOCOL — screenshot lightbox ────────────────────────────────────
function openVisionLightbox(title) {
const lb = document.getElementById('vision-lightbox');
document.getElementById('vision-lb-title').textContent = title || '◈ VISION PROTOCOL';
document.getElementById('vision-lb-img').style.display = 'none';
document.getElementById('vision-lb-img').src = '';
document.getElementById('vision-lb-analysis').textContent = '';
document.getElementById('vision-lb-spinner').style.display = 'block';
lb.classList.add('open');
}
function closeVisionLightbox() {
document.getElementById('vision-lightbox').classList.remove('open');
}
async function agentScreenshot(hostname) {
openVisionLightbox('◈ VISION PROTOCOL — ' + hostname.toUpperCase());
const arcRes = await api('arc?action=job_create', 'POST', {
type: 'screenshot',
payload: {agent: hostname, analyze: true},
priority: 8,
}).catch(() => null);
if (!arcRes || !arcRes.job_id) {
document.getElementById('vision-lb-spinner').style.display = 'none';
document.getElementById('vision-lb-analysis').textContent = 'Failed to submit screenshot job — Arc Reactor may be offline.';
return;
}
// Poll for result
const jobId = arcRes.job_id;
let tries = 0;
const poll = async () => {
tries++;
const job = await api('arc?action=job_get&id=' + jobId).catch(() => null);
if (job && job.status === 'done') {
const r = job.result || {};
document.getElementById('vision-lb-spinner').style.display = 'none';
if (r.has_image && r.screenshot_id) {
// Fetch full screenshot with image
const full = await api('arc?action=screenshot_get&id=' + r.screenshot_id).catch(() => null);
if (full && full.image_b64) {
const img = document.getElementById('vision-lb-img');
img.src = 'data:image/png;base64,' + full.image_b64;
img.style.display = 'block';
}
}
document.getElementById('vision-lb-analysis').textContent =
r.analysis || (r.has_image ? 'Screenshot captured — no analysis available.' : JSON.stringify(r.snapshot || r, null, 2));
} else if (job && job.status === 'failed') {
document.getElementById('vision-lb-spinner').style.display = 'none';
document.getElementById('vision-lb-analysis').textContent = 'Screenshot failed: ' + (job.error || 'Unknown error');
} else if (tries < 30) {
setTimeout(poll, 2000);
} else {
document.getElementById('vision-lb-spinner').style.display = 'none';
document.getElementById('vision-lb-analysis').textContent = 'Timed out waiting for screenshot.';
}
};
setTimeout(poll, 2000);
}
async function agentSysinfo(hostname) {
openVisionLightbox('⚡ FIELD SYSINFO — ' + hostname.toUpperCase());
const arcRes = await api('arc?action=job_create', 'POST', {
type: 'sysinfo',
payload: {agent: hostname, analyze: true},
priority: 7,
}).catch(() => null);
if (!arcRes || !arcRes.job_id) {
document.getElementById('vision-lb-spinner').style.display = 'none';
document.getElementById('vision-lb-analysis').textContent = 'Failed to submit sysinfo job.';
return;
}
const jobId = arcRes.job_id;
let tries = 0;
const poll = async () => {
tries++;
const job = await api('arc?action=job_get&id=' + jobId).catch(() => null);
if (job && job.status === 'done') {
const r = job.result || {};
document.getElementById('vision-lb-spinner').style.display = 'none';
const snap = r.snapshot || {};
const snapText = Object.entries(snap)
.filter(([k]) => !['success','screenshot_available','snapshot_type'].includes(k))
.map(([k,v]) => `${k.toUpperCase().replace(/_/g,' ')}: ${Array.isArray(v) ? v.join('\n ') : v}`)
.join('\n');
document.getElementById('vision-lb-analysis').textContent =
(r.analysis ? r.analysis + '\n\n─────────────────────\n\n' : '') + (snapText || JSON.stringify(r, null, 2));
} else if (job && job.status === 'failed') {
document.getElementById('vision-lb-spinner').style.display = 'none';
document.getElementById('vision-lb-analysis').textContent = 'Sysinfo failed: ' + (job.error || 'Unknown error');
} else if (tries < 20) {
setTimeout(poll, 2000);
} else {
document.getElementById('vision-lb-spinner').style.display = 'none';
document.getElementById('vision-lb-analysis').textContent = 'Timed out.';
}
};
setTimeout(poll, 2000);
}
document.addEventListener('keydown', e => {
if (e.key === 'Escape') closeVisionLightbox();
});
</script> </script>
<!-- VISION LIGHTBOX -->
<div id="vision-lightbox">
<div id="vision-lb-header">
<span id="vision-lb-title">◈ VISION PROTOCOL</span>
<button id="vision-lb-close" onclick="closeVisionLightbox()">✕ CLOSE</button>
</div>
<div id="vision-lb-spinner">● SCANNING...</div>
<img id="vision-lb-img" alt="Agent Screenshot" style="display:none">
<pre id="vision-lb-analysis"></pre>
</div>
</body> </body>
</html> </html>