myron/jarvis
1
0
Fork 0
mirror of https://github.com/myronblair/jarvis synced 2026-06-30 17:50:23 -05:00
Code Issues Packages Projects Releases Wiki Activity

Phase 4: Vision Protocol — screenshot + Claude vision

Browse Source 
- reactor.py: v4.0.0; adds screenshot, vision, sysinfo handlers;
  _dispatch_agent_command() shared helper; FastAPI /screenshots endpoints
- jarvis-agent.py: v3.0; screenshot command handler (scrot/import/fbcat/
  ImageMagick render fallback); sysinfo command returns structured snapshot;
  detect_capabilities() advertises screenshot + sysinfo caps
- chat.php: Tier 0.9c detects screenshot (show screen on X, screenshot X)
  and sysinfo (check status of X) voice/text commands
- arc.php: screenshots, screenshot_get, screenshot_delete actions
- index.html: VISION PROTOCOL lightbox overlay; SCREENSHOT + SYSINFO
  buttons on each online agent card; keyboard Escape to close
- admin/index.php: VISION PROTOCOL tab under ARC REACTOR nav; gallery view
  with image thumbnails + analysis; take screenshot modal; purge action
This commit is contained in:
Myron Blair
2026-06-11 04:42:21 +00:00
parent 068aff27b4
commit 56c9e2d914
5 changed files with 576 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/endpoints/chat.php
+34 -2
View File
@@ -1063,7 +1063,7 @@ if (!$reply && preg_match('/\b(news|headlines|latest|what.?s happening|current e
}
}
// ── Tier 0.9: Intel Protocol — research, tool_loop, gmail_triage, remote_exec ─
// ── Tier 0.9: Arc Protocols — research, triage, remote_exec, screenshot, sysinfo ─
$arcJobId = null;
// Helper: submit job to Arc Reactor
@@ -1158,7 +1158,39 @@ if (!$reply) {
}
}
// ── Tier 0.9c: Intel Protocol — research & tool_loop detection ────────────
// ── Tier 0.9c: Vision Protocol — screenshot, sysinfo, vision detection ───────
if (!$reply) {
// Screenshot patterns
$screenshotMatch = null;
if (preg_match('/^(?:jarvis[,\s]+)?(?:show\s+(?:me\s+)?(?:the\s+)?screen\s+(?:on|of|from)|screenshot\s+(?:of\s+)?|grab\s+(?:a\s+)?(?:screenshot|screen\s+cap)\s+(?:of\s+|from\s+)?|what(?:\'s|\s+is)\s+(?:on|showing\s+on)\s+(?:the\s+)?screen\s+(?:on|of))\s+(.+)/i', $message, $mm)) {
$screenshotMatch = trim($mm[1]);
$arcRes = arcSubmitJob('screenshot', ['agent' => $screenshotMatch, 'analyze' => true], $sessionId);
if (isset($arcRes['job_id'])) {
$arcJobId = $arcRes['job_id'];
$reply = "◈ VISION PROTOCOL ACTIVATED — Capturing screen on **{$screenshotMatch}** (Job #{$arcJobId}). I'll analyze what I see and report back, {$userAddr}.";
$source = 'arc:screenshot';
} else {
$reply = "Vision Protocol is offline, {$userAddr}. Arc Reactor may be unavailable.";
$source = 'arc:offline';
}
}
// Sysinfo patterns
elseif (preg_match('/^(?:jarvis[,\s]+)?(?:(?:what(?:\'s|\s+is)\s+(?:the\s+)?status\s+of|check\s+(?:the\s+)?(?:health|status)\s+of|how\s+is)\s+(.+)|system\s+(?:info|status|snapshot)\s+(?:on|from|for)\s+(.+))/i', $message, $mm)) {
$agentName = trim($mm[1] ?? $mm[2] ?? '');
// Only fire for explicit agent references (has a name), not generic questions
if ($agentName && strlen($agentName) > 2 &&
!preg_match('/\b(?:weather|today|things|everything|jarvis|yourself)\b/i', $agentName)) {
$arcRes = arcSubmitJob('sysinfo', ['agent' => $agentName, 'analyze' => true], $sessionId);
if (isset($arcRes['job_id'])) {
$arcJobId = $arcRes['job_id'];
$reply = "◈ FIELD PROTOCOL — Running system health check on **{$agentName}** (Job #{$arcJobId}). Snapshot incoming, {$userAddr}.";
$source = 'arc:sysinfo';
}
}
}
}
// ── Tier 0.9e: Intel Protocol — research & tool_loop detection ────────────
$intelPatterns = [
'/^(?:jarvis[,\s]+)?(?:research|investigate|deep[- ]dive|deep dive)\s+(.+)/i' => 'research',
'/^(?:jarvis[,\s]+)?(?:look\s+(?:up|into)|find\s+out\s+(?:about)?)\s+(.+)/i' => 'research',