mirror of
https://github.com/myronblair/jarvis
synced 2026-06-30 17:50:23 -05:00
Phase 4: Vision Protocol — screenshot + Claude vision
- reactor.py: v4.0.0; adds screenshot, vision, sysinfo handlers; _dispatch_agent_command() shared helper; FastAPI /screenshots endpoints - jarvis-agent.py: v3.0; screenshot command handler (scrot/import/fbcat/ ImageMagick render fallback); sysinfo command returns structured snapshot; detect_capabilities() advertises screenshot + sysinfo caps - chat.php: Tier 0.9c detects screenshot (show screen on X, screenshot X) and sysinfo (check status of X) voice/text commands - arc.php: screenshots, screenshot_get, screenshot_delete actions - index.html: VISION PROTOCOL lightbox overlay; SCREENSHOT + SYSINFO buttons on each online agent card; keyboard Escape to close - admin/index.php: VISION PROTOCOL tab under ARC REACTOR nav; gallery view with image thumbnails + analysis; take screenshot modal; purge action
This commit is contained in:
@@ -550,6 +550,48 @@ if ($action) {
|
||||
$raw = curl_exec($ch); curl_close($ch);
|
||||
j(json_decode($raw, true) ?: ['error' => 'Arc Reactor unreachable']);
|
||||
|
||||
// ── VISION PROTOCOL ──────────────────────────────────────────────────
|
||||
case 'vision_list':
|
||||
$limit = min((int)($_GET['limit'] ?? 30), 100);
|
||||
$agent = $_GET['agent'] ?? '';
|
||||
$url = 'http://127.0.0.1:7474/screenshots?' . http_build_query(array_filter(['limit'=>$limit,'agent'=>$agent]));
|
||||
$ch = curl_init($url);
|
||||
curl_setopt_array($ch, [CURLOPT_RETURNTRANSFER=>true, CURLOPT_TIMEOUT=>5]);
|
||||
$raw = curl_exec($ch); curl_close($ch);
|
||||
j(json_decode($raw, true) ?: []);
|
||||
|
||||
case 'vision_get':
|
||||
$id = (int)($_GET['id'] ?? 0); if (!$id) bad('Missing id');
|
||||
$ch = curl_init('http://127.0.0.1:7474/screenshots/' . $id);
|
||||
curl_setopt_array($ch, [CURLOPT_RETURNTRANSFER=>true, CURLOPT_TIMEOUT=>5]);
|
||||
$raw = curl_exec($ch); curl_close($ch);
|
||||
j(json_decode($raw, true) ?: ['error'=>'not found']);
|
||||
|
||||
case 'vision_delete':
|
||||
$id = (int)($_GET['id'] ?? 0); if (!$id) bad('Missing id');
|
||||
$ch = curl_init('http://127.0.0.1:7474/screenshots/' . $id);
|
||||
curl_setopt_array($ch, [CURLOPT_RETURNTRANSFER=>true, CURLOPT_TIMEOUT=>5, CURLOPT_CUSTOMREQUEST=>'DELETE']);
|
||||
$raw = curl_exec($ch); curl_close($ch);
|
||||
j(json_decode($raw, true) ?: ['ok'=>true]);
|
||||
|
||||
case 'vision_screenshot':
|
||||
$agent = trim($_GET['agent'] ?? ''); if (!$agent) bad('Missing agent');
|
||||
$analyze = ($_GET['analyze'] ?? '1') !== '0';
|
||||
$ch = curl_init('http://127.0.0.1:7474/job');
|
||||
curl_setopt_array($ch, [
|
||||
CURLOPT_RETURNTRANSFER => true, CURLOPT_TIMEOUT => 5, CURLOPT_POST => true,
|
||||
CURLOPT_POSTFIELDS => json_encode(['type'=>'screenshot','payload'=>['agent'=>$agent,'analyze'=>$analyze],'priority'=>8,'created_by'=>'admin']),
|
||||
CURLOPT_HTTPHEADER => ['Content-Type: application/json'],
|
||||
]);
|
||||
$raw = curl_exec($ch); curl_close($ch);
|
||||
j(json_decode($raw, true) ?: ['error'=>'Arc Reactor unreachable']);
|
||||
|
||||
case 'vision_purge':
|
||||
$ch = curl_init('http://127.0.0.1:7474/screenshots/purge');
|
||||
curl_setopt_array($ch, [CURLOPT_RETURNTRANSFER=>true, CURLOPT_TIMEOUT=>5, CURLOPT_CUSTOMREQUEST=>'DELETE']);
|
||||
$raw = curl_exec($ch); curl_close($ch);
|
||||
j(json_decode($raw, true) ?: ['ok'=>true]);
|
||||
|
||||
case 'users_list':
|
||||
j(JarvisDB::query('SELECT id,username,display_name,last_seen,created_at FROM users ORDER BY username'));
|
||||
|
||||
@@ -789,6 +831,7 @@ select.filter-sel:focus{border-color:var(--cyan)}
|
||||
<div class="nav-item" data-tab="calendar" onclick="nav(this)">🗓 CALENDAR SYNC</div>
|
||||
<div class="nav-section">ARC REACTOR</div>
|
||||
<div class="nav-item" data-tab="arc" onclick="nav(this)">⚡ ARC REACTOR</div>
|
||||
<div class="nav-item" data-tab="vision" onclick="nav(this)">◈ VISION PROTOCOL</div>
|
||||
<div class="nav-section">INFO</div>
|
||||
<div class="nav-item" data-tab="sites" onclick="nav(this)">SITES</div>
|
||||
<div class="nav-item" data-tab="users" onclick="nav(this)">USERS</div>
|
||||
@@ -1085,6 +1128,25 @@ select.filter-sel:focus{border-color:var(--cyan)}
|
||||
<div class="tbl-wrap" id="arc-jobs-tbl"><div class="loading">INITIALIZING...</div></div>
|
||||
</div>
|
||||
|
||||
<!-- VISION PROTOCOL -->
|
||||
<div class="tab" id="tab-vision">
|
||||
<div class="page-title">◈ VISION PROTOCOL — FIELD SCREENSHOTS</div>
|
||||
|
||||
<div style="display:flex;gap:10px;margin-bottom:16px;align-items:center;flex-wrap:wrap">
|
||||
<button class="btn btn-sm btn-green" onclick="visionRunScreenshot()">◈ TAKE SCREENSHOT</button>
|
||||
<button class="btn btn-sm" onclick="loadVision()">↻ REFRESH</button>
|
||||
<select id="vision-agent-filter" class="inp" style="width:auto;padding:4px 8px;font-size:0.65rem" onchange="loadVision()">
|
||||
<option value="">ALL AGENTS</option>
|
||||
</select>
|
||||
<button class="btn btn-sm" onclick="visionPurge()" style="margin-left:auto;opacity:0.7">PURGE OLD</button>
|
||||
<div id="vision-count" style="font-family:var(--mono);font-size:0.65rem;color:var(--dim)"></div>
|
||||
</div>
|
||||
|
||||
<div id="vision-gallery" style="display:grid;grid-template-columns:repeat(auto-fill,minmax(280px,1fr));gap:14px">
|
||||
<div class="loading">LOADING SCREENSHOTS...</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- GMAIL TRIAGE -->
|
||||
<div class="tab" id="tab-triage">
|
||||
<div class="page-title">◈ COMMS PROTOCOL — GMAIL TRIAGE</div>
|
||||
@@ -1219,6 +1281,7 @@ function loadTab(tab) {
|
||||
users: loadUsers,
|
||||
email: ()=>{ loadEmailInbox(); loadEmailActionItems(); },
|
||||
triage: loadTriage,
|
||||
vision: loadVision,
|
||||
tasks: loadTasks,
|
||||
appointments: loadAppts,
|
||||
calendar: loadCalFeeds,
|
||||
@@ -2110,6 +2173,122 @@ function emailDismiss(id) {
|
||||
apiPost('email_dismiss',{id},()=>{ toast('Dismissed','ok'); loadEmailActionItems(); });
|
||||
}
|
||||
|
||||
// ── VISION PROTOCOL ──────────────────────────────────────────────────────────
|
||||
let _visionAgents = [];
|
||||
|
||||
async function loadVision() {
|
||||
const gallery = document.getElementById('vision-gallery');
|
||||
if (!gallery) return;
|
||||
gallery.innerHTML = '<div class="loading">LOADING...</div>';
|
||||
|
||||
const filter = document.getElementById('vision-agent-filter')?.value || '';
|
||||
const shots = await api('vision_list', {limit: 30, agent: filter});
|
||||
|
||||
// Populate agent filter dropdown
|
||||
const select = document.getElementById('vision-agent-filter');
|
||||
if (select && Array.isArray(shots) && shots.length) {
|
||||
const agents = [...new Set(shots.map(s => s.hostname).filter(Boolean))];
|
||||
_visionAgents = agents;
|
||||
const current = select.value;
|
||||
select.innerHTML = '<option value="">ALL AGENTS</option>' +
|
||||
agents.map(a => `<option value="${esc(a)}"${a===current?' selected':''}>${esc(a)}</option>`).join('');
|
||||
}
|
||||
|
||||
document.getElementById('vision-count').textContent = Array.isArray(shots) ? shots.length + ' SCREENSHOTS' : '';
|
||||
|
||||
if (!Array.isArray(shots) || !shots.length) {
|
||||
gallery.innerHTML = '<div class="loading">No screenshots yet. Click "TAKE SCREENSHOT" to capture a field station.</div>';
|
||||
return;
|
||||
}
|
||||
|
||||
gallery.innerHTML = shots.map(s => {
|
||||
const ts = ts(s.created_at);
|
||||
const has = s.file_size > 0;
|
||||
const meth = (s.method || 'unknown').toUpperCase();
|
||||
const dim = s.width && s.height ? `${s.width}×${s.height}` : '';
|
||||
const analysis = (s.vision_analysis || '').substring(0, 180);
|
||||
return `<div style="background:rgba(0,212,255,0.03);border:1px solid var(--border);border-radius:4px;overflow:hidden">
|
||||
<div style="background:rgba(0,212,255,0.06);padding:8px 10px;display:flex;align-items:center;gap:8px">
|
||||
<span style="font-family:var(--mono);font-size:0.65rem;flex:1;white-space:nowrap;overflow:hidden;text-overflow:ellipsis">${esc(s.hostname||'unknown')}</span>
|
||||
<span style="font-family:var(--mono);font-size:0.55rem;color:var(--dim)">${meth}</span>
|
||||
<button class="btn btn-xs" onclick="visionViewScreenshot(${s.id})" style="border-color:var(--cyan);color:var(--cyan)">VIEW</button>
|
||||
<button class="btn btn-xs" onclick="visionDeleteShot(${s.id})">✗</button>
|
||||
</div>
|
||||
<div style="padding:8px 10px">
|
||||
${has ? `<div style="background:#060a0e;border:1px solid var(--border);border-radius:3px;padding:5px;margin-bottom:8px;cursor:pointer;font-family:var(--mono);font-size:0.6rem;color:var(--text-dim);text-align:center" onclick="visionViewScreenshot(${s.id})">
|
||||
◈ ${dim ? dim + ' · ' : ''}${Math.round((s.file_size||0)/1024)}KB IMAGE
|
||||
</div>` : '<div style="font-size:0.6rem;color:var(--dim);margin-bottom:6px;font-family:var(--mono)">TEXT SNAPSHOT ONLY</div>'}
|
||||
${analysis ? `<div style="font-size:0.62rem;line-height:1.5;color:var(--text-dim)">${esc(analysis)}${s.vision_analysis?.length>180?'…':''}</div>` : ''}
|
||||
<div style="font-family:var(--mono);font-size:0.55rem;color:var(--border2);margin-top:6px">${ts}</div>
|
||||
</div>
|
||||
</div>`;
|
||||
}).join('');
|
||||
}
|
||||
|
||||
async function visionViewScreenshot(id) {
|
||||
const d = await api('vision_get', {id});
|
||||
if (d.error) { toast('Error: ' + d.error, 'err'); return; }
|
||||
|
||||
const imgHtml = d.image_b64
|
||||
? `<img src="data:image/png;base64,${d.image_b64}" style="max-width:100%;border:1px solid var(--border);border-radius:3px;margin-bottom:12px">`
|
||||
: '<div style="color:var(--dim);font-family:var(--mono);font-size:0.65rem;padding:12px 0">No image data — text snapshot only</div>';
|
||||
|
||||
openModal('◈ VISION — ' + esc(d.hostname||''), `
|
||||
<div style="font-size:0.6rem;color:var(--dim);margin-bottom:10px;font-family:var(--mono)">
|
||||
METHOD: ${esc(d.method||'')} · ${d.width&&d.height?d.width+'×'+d.height+' · ':''} ${Math.round((d.file_size||0)/1024)}KB · ${ts(d.created_at)}
|
||||
</div>
|
||||
${imgHtml}
|
||||
${d.vision_analysis ? `<div style="font-size:0.55rem;letter-spacing:2px;color:var(--dim);margin-bottom:6px">VISION ANALYSIS</div>
|
||||
<pre style="white-space:pre-wrap;font-size:0.65rem;line-height:1.6;color:var(--text);background:rgba(0,212,255,0.04);border:1px solid var(--border);padding:10px;border-radius:3px;max-height:300px;overflow-y:auto">${esc(d.vision_analysis)}</pre>` : ''}
|
||||
`, null, null);
|
||||
document.getElementById('modalSave').style.display = 'none';
|
||||
}
|
||||
|
||||
async function visionRunScreenshot() {
|
||||
// Pick agent from dropdown or prompt
|
||||
const agents = _visionAgents;
|
||||
if (!agents.length) {
|
||||
toast('No agents online — check AGENTS tab', 'err'); return;
|
||||
}
|
||||
// Build select for agent
|
||||
openModal('◈ TAKE SCREENSHOT', `
|
||||
<div style="margin-bottom:14px">
|
||||
<label style="font-size:0.65rem;color:var(--dim);display:block;margin-bottom:6px">SELECT FIELD AGENT</label>
|
||||
<select id="vision-agent-sel" class="inp" style="width:100%">
|
||||
${agents.map(a => `<option value="${esc(a)}">${esc(a)}</option>`).join('')}
|
||||
</select>
|
||||
</div>
|
||||
<div>
|
||||
<label style="font-size:0.65rem;color:var(--dim);display:flex;align-items:center;gap:8px;cursor:pointer">
|
||||
<input type="checkbox" id="vision-analyze-chk" checked> Run Claude vision analysis
|
||||
</label>
|
||||
</div>
|
||||
`, async () => {
|
||||
const agent = document.getElementById('vision-agent-sel')?.value || '';
|
||||
const analyze = document.getElementById('vision-analyze-chk')?.checked ? '1' : '0';
|
||||
const d = await api('vision_screenshot', {agent, analyze});
|
||||
if (d.job_id) {
|
||||
toast('Screenshot job started — Job #' + d.job_id, 'ok');
|
||||
setTimeout(() => loadVision(), 5000);
|
||||
} else {
|
||||
toast('Failed: ' + (d.error || 'Arc offline'), 'err');
|
||||
}
|
||||
closeModal();
|
||||
}, 'CAPTURE');
|
||||
}
|
||||
|
||||
async function visionDeleteShot(id) {
|
||||
await api('vision_delete', {id});
|
||||
toast('Deleted', 'ok');
|
||||
loadVision();
|
||||
}
|
||||
|
||||
async function visionPurge() {
|
||||
await api('vision_purge');
|
||||
toast('Old screenshots purged', 'ok');
|
||||
loadVision();
|
||||
}
|
||||
|
||||
// ── GMAIL TRIAGE ─────────────────────────────────────────────────────────────
|
||||
const _TRIAGE_COLORS = {urgent:'var(--red)',action:'var(--orange)',reply:'var(--cyan)',meeting:'#a78bfa',info:'var(--text-dim)',promo:'rgba(255,255,255,0.25)',spam:'rgba(255,255,255,0.15)'};
|
||||
|
||||
|
||||
Reference in New Issue
Block a user