From b19e8e1b253da1d5102157f1cbf7f7187aa9523a Mon Sep 17 00:00:00 2001 From: Myron Blair Date: Thu, 11 Jun 2026 21:37:26 +0000 Subject: [PATCH] =?UTF-8?q?Phase=202:=20facts=5Fcollector=20TTL=20guards?= =?UTF-8?q?=20=E2=80=94=20reduce=20external=20polling?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Proxmox: skip if data < 10 min old (was: every 3 min unconditionally) - Ollama: skip if data < 15 min old (model list rarely changes) - Site health: skip if data < 5 min old (was: 7 HTTP calls every 3 min) - Home Assistant: removed entirely (HA agent pushes 212 entities every 30s) Fast local reads (CPU/mem/network pings) still run every 3 min. External HTTP calls now fire only when data is actually stale. Saves ~140 site-check HTTP calls/hour and ~60 Proxmox API calls/hour in steady state. --- api/endpoints/facts_collector.php | 141 ++++++------------------------ 1 file changed, 26 insertions(+), 115 deletions(-) diff --git a/api/endpoints/facts_collector.php b/api/endpoints/facts_collector.php index 64a0c2c..646cae7 100644 --- a/api/endpoints/facts_collector.php +++ b/api/endpoints/facts_collector.php @@ -19,6 +19,18 @@ function collect_all(): array { $results = []; $ttl = 300; // 5-minute TTL on live facts + // Returns true if a fact category has been updated within $secs seconds. + // Prevents expensive external calls when data is still fresh. + $fresh = function(string $cat, int $secs): bool { + $row = JarvisDB::query( + 'SELECT updated_at FROM kb_facts WHERE fact_category=? ORDER BY updated_at DESC LIMIT 1', + [$cat] + ); + if (empty($row[0]['updated_at'])) return false; + return (time() - strtotime($row[0]['updated_at'])) < $secs; + }; + + // ── System ──────────────────────────────────────────────────────────── try { $stat1 = file_get_contents('/proc/stat'); @@ -94,8 +106,10 @@ function collect_all(): array { $results['network'] = 'error: ' . $e->getMessage(); } - // ── Proxmox ─────────────────────────────────────────────────────────── - try { + // ── Proxmox (TTL 10 min) ───────────────────────────────────────────── + if ($fresh('proxmox', 600)) { + $results['proxmox'] = 'skipped (fresh)'; + } else try { if (defined('PROXMOX_TOKEN_ID') && PROXMOX_TOKEN_ID) { $base = 'https://orbisne.fortiddns.com:' . PROXMOX_PORT . '/api2/json'; $auth = 'Authorization: PVEAPIToken=' . PROXMOX_USER . '!' . PROXMOX_TOKEN_ID . '=' . PROXMOX_TOKEN_VAL; @@ -126,116 +140,9 @@ function collect_all(): array { $results['proxmox'] = 'error: ' . $e->getMessage(); } - // ── Home Assistant ──────────────────────────────────────────────────── - try { - if (defined('HA_URL') && defined('HA_TOKEN') && HA_TOKEN !== 'YOUR_HA_TOKEN_HERE') { - $haUrl = HA_URL; - $haToken = HA_TOKEN; - $haHdr = ['Authorization: Bearer ' . $haToken, 'Content-Type: application/json']; + // ── Home Assistant — skipped (HA agent pushes entities every 30s) ──── + $results['ha'] = 'skipped (agent push active)'; - // Fetch all entity states - $ch = curl_init($haUrl . '/api/states'); - curl_setopt_array($ch, [ - CURLOPT_RETURNTRANSFER => true, - CURLOPT_HTTPHEADER => $haHdr, - CURLOPT_TIMEOUT => 12, - CURLOPT_CONNECTTIMEOUT => 5, - ]); - $resp = curl_exec($ch); - $code = curl_getinfo($ch, CURLINFO_HTTP_CODE); - curl_close($ch); - - if ($code === 200) { - $allStates = json_decode($resp, true) ?? []; - - // Domains to index for control - $controlDomains = ['light','switch','input_boolean','climate','cover','fan', - 'scene','script','lawn_mower','vacuum','media_player']; - // Switch keywords to skip (camera/HACS settings, not real devices) - $skipKeywords = ['pre_release','_record','_ftp_','_push_','_hub_ringtone', - '_siren_on','_email_on','_manual_record','_infrared_', - 'do_not_disturb','matter_server','zerotier','mariadb', - 'spotify','file_editor','ssh_web','uptime_kuma', - 'adguard_home_','adguard_protection','adguard_parental', - 'adguard_safe','adguard_filter','adguard_query', - 'assist_microphone','folding_home','music_assistant', - 'get_hacs','mealie','mosquitto','social_to', - 'motion_detection','front_yard_record','down_hill_record', - 'camera1_record','back_yard_record','nvr_']; - - $entityMap = []; - $statusCount = ['online' => 0, 'offline' => 0, 'unavailable' => 0]; - - foreach ($allStates as $s) { - $eid = $s['entity_id']; - $domain = explode('.', $eid)[0]; - $name = $s['attributes']['friendly_name'] ?? $eid; - $state = $s['state']; - - if (!in_array($domain, $controlDomains)) continue; - - // Skip camera/HACS internals for switches - if ($domain === 'switch') { - $skip = false; - foreach ($skipKeywords as $kw) { - if (strpos($eid, $kw) !== false) { $skip = true; break; } - } - if ($skip) continue; - } - - $entityMap[$eid] = ['name' => $name, 'state' => $state, 'domain' => $domain]; - - if ($state === 'unavailable' || $state === 'unknown') { - $statusCount['unavailable']++; - } elseif (in_array($state, ['on','open','playing','mowing','home','active','idle'])) { - $statusCount['online']++; - } elseif (in_array($state, ['off','closed','paused','docked','away'])) { - $statusCount['offline']++; - } - } - - // Store entity map as JSON for chat.php to use - KBEngine::storeFact('ha', 'entity_map', json_encode($entityMap), 'ha', 270); - KBEngine::storeFact('ha', 'entity_count', count($entityMap), 'ha', $ttl); - KBEngine::storeFact('ha', 'online_count', $statusCount['online'], 'ha', $ttl); - KBEngine::storeFact('ha', 'offline_count', $statusCount['offline'], 'ha', $ttl); - KBEngine::storeFact('ha', 'unavail_count', $statusCount['unavailable'], 'ha', $ttl); - KBEngine::storeFact('ha', 'ha_status', 'online', 'ha', $ttl); - - // Store individual sensor facts - $sensorDomains = ['sensor','binary_sensor','weather']; - $interestingPatterns = ['temperature','humidity','battery','power','energy', - 'voltage','current','illuminance','co2','pm25']; - foreach ($allStates as $s) { - $domain = explode('.', $s['entity_id'])[0]; - if (!in_array($domain, $sensorDomains)) continue; - $eid = $s['entity_id']; - foreach ($interestingPatterns as $pat) { - if (strpos($eid, $pat) !== false) { - $name = $s['attributes']['friendly_name'] ?? $eid; - $unit = $s['attributes']['unit_of_measurement'] ?? ''; - KBEngine::storeFact('ha_sensors', $eid, - $s['state'] . ($unit ? " {$unit}" : ''), 'ha', $ttl); - break; - } - } - } - - $results['ha'] = sprintf('ok (%d entities, %d on, %d off, %d unavail)', - count($entityMap), $statusCount['online'], - $statusCount['offline'], $statusCount['unavailable']); - } else { - KBEngine::storeFact('ha', 'ha_status', 'unreachable', 'ha', $ttl); - $results['ha'] = "unreachable (HTTP {$code})"; - } - } else { - KBEngine::storeFact('ha', 'ha_status', 'token not configured', 'ha', $ttl); - $results['ha'] = 'skipped (no token)'; - } - } catch (Exception $e) { - KBEngine::storeFact('ha', 'ha_status', 'error', 'ha', $ttl); - $results['ha'] = 'error: ' . $e->getMessage(); - } // ── Digital Ocean ───────────────────────────────────────────────────── try { @@ -247,8 +154,10 @@ function collect_all(): array { $results['do_server'] = 'error: ' . $e->getMessage(); } - // ── Ollama ──────────────────────────────────────────────────────────── - try { + // ── Ollama (TTL 15 min) ─────────────────────────────────────────────── + if ($fresh('ollama', 900)) { + $results['ollama'] = 'skipped (fresh)'; + } else try { $ollamaHost = defined('OLLAMA_HOST') ? OLLAMA_HOST : 'http://10.48.200.95:11434'; $ch = curl_init($ollamaHost . '/api/tags'); curl_setopt_array($ch, [CURLOPT_RETURNTRANSFER => true, CURLOPT_TIMEOUT => 5]); @@ -278,8 +187,10 @@ function collect_all(): array { $results['ollama'] = 'error: ' . $e->getMessage(); } - // ── Site Health ─────────────────────────────────────────────────────── - try { + // ── Site Health (TTL 5 min) ─────────────────────────────────────────── + if ($fresh('sites', 300)) { + $results['sites'] = 'skipped (fresh)'; + } else try { $sites = [ 'jarvis' => 'https://jarvis.orbishosting.com', 'tomsjavajive' => 'https://tomsjavajive.com',