""" ECO-ROVER NEXUS - GEMINI VISION + EMAIL ALERTS Raspberry Pi 3B / 1 GB RAM One Gemini call analyses the whole frame and returns structured JSON: * is there a human? (with confidence) * is there a plant? (with confidence) * which plant species? (common + scientific name) * is the plant diseased? (disease name, severity, symptoms) * what should the grower do? (actionable advice) WHY THIS IS RATE LIMITED ------------------------ The Gemini free tier allows on the order of a few hundred to ~1500 requests per DAY depending on model and current Google policy. Calling it every 5 seconds would be ~17,000 calls/day and would exhaust the quota within minutes and then fail all day. So this module is deliberately frugal: 1. MOTION GATE - a cheap local frame-difference check. Gemini is only called when the scene actually changes. 2. MIN INTERVAL - a hard floor between any two calls. 3. DAILY BUDGET - a hard cap; when spent, it stops until midnight. 4. HEALTH SCAN - a few scheduled plant/disease checks per day. Everything degrades safely: no key, no network or quota exhausted means the dashboard keeps running and simply reports vision as unavailable. SETUP ----- export GEMINI_API_KEY="your_key_from_aistudio.google.com" export ECO_SMTP_USER="you@gmail.com" export ECO_SMTP_PASS="16charAppPassword" export ECO_ALERT_TO="destination@example.com" """ import os import ssl import json import time import queue import base64 import smtplib import threading import urllib.error import urllib.request from email.message import EmailMessage from datetime import datetime, date try: import cv2 except ImportError: cv2 = None try: import numpy as np except ImportError: np = None # ============================================================ # CONFIGURATION # ============================================================ BASE_DIR = os.path.expanduser("~/regen_ai") SNAPSHOT_DIR = os.path.join(BASE_DIR, "snapshots") STATE_FILE = os.path.join(BASE_DIR, "gemini_usage.json") GEMINI_API_KEY = os.environ.get("GEMINI_API_KEY", "") # Tried in order. If one is retired or over quota the next is used. GEMINI_MODELS = [ "gemini-2.5-flash", "gemini-2.0-flash", "gemini-2.5-flash-lite", "gemini-flash-latest", ] API_ROOT = "https://generativelanguage.googleapis.com/v1beta/models" # ---- Call pacing ------------------------------------------------- # ANALYSIS_INTERVAL_SECONDS is the headline setting: how often Gemini # looks at the scene. # # 10.1s = 8,554 calls/day <-- current setting, beyond the free tier # 30s = 2,880 calls/day # 60s = 1,440 calls/day <-- about the most generous free tier # 300s = 288 calls/day <-- comfortable on a small free tier # # The free Gemini tier is roughly 10-15 requests/MINUTE and only a few # hundred to ~1500 requests/DAY. At 10.1s you use ~5.9 calls/minute, # which is inside the per-minute cap, but ~8,554/day still exceeds the # daily cap - expect roughly 42 min to 4.2 h before quota runs out. # # AUTO_BACKOFF (below) protects you: on a 429 the interval doubles # instead of hammering a dead quota, and recovers when calls succeed. ANALYSIS_INTERVAL_SECONDS = float( os.environ.get("ECO_GEMINI_INTERVAL", "10.1")) # Hard ceiling on calls per day. 0 disables the cap entirely. # Default 0 because an explicit interval is now the primary control. DAILY_CALL_BUDGET = int(os.environ.get("ECO_GEMINI_DAILY_BUDGET", "0")) # When a 429 (rate limit / quota) arrives, multiply the interval by this # up to MAX_BACKOFF_SECONDS, so the Pi stops burning failed requests. AUTO_BACKOFF = os.environ.get("ECO_GEMINI_BACKOFF", "1") not in ("0", "false") BACKOFF_MULTIPLIER = 2.0 MAX_BACKOFF_SECONDS = 900.0 # 15 minutes # Optional motion gate. Default OFF so the interval is honoured exactly. # Turn on with ECO_GEMINI_MOTION_GATE=1 to skip calls on a static scene. USE_MOTION_GATE = os.environ.get("ECO_GEMINI_MOTION_GATE", "0") == "1" MOTION_THRESHOLD = 0.020 HEALTH_SCAN_HOURS = [8, 13, 18] # Alerting PERSON_ALERT_COOLDOWN = 300 # 5 min between person emails DISEASE_ALERT_COOLDOWN = 21600 # 6 h between disease emails PERSON_MIN_CONFIDENCE = 60 # percent DISEASE_MIN_CONFIDENCE = 55 # percent REQUEST_TIMEOUT = 45 # Email SMTP_HOST = os.environ.get("ECO_SMTP_HOST", "smtp.gmail.com") SMTP_PORT = int(os.environ.get("ECO_SMTP_PORT", "587")) SMTP_USER = os.environ.get("ECO_SMTP_USER", "") SMTP_PASS = os.environ.get("ECO_SMTP_PASS", "") ALERT_TO = os.environ.get("ECO_ALERT_TO", "") EMAIL_ENABLED = bool(SMTP_USER and SMTP_PASS and ALERT_TO) GEMINI_ENABLED = bool(GEMINI_API_KEY) # ============================================================ # THE PROMPT # ============================================================ ANALYSIS_PROMPT = """You are the vision system of an autonomous farm monitoring robot. Analyse this camera frame and report exactly what you can see. Be strict and factual. If something is not clearly visible, say so rather than guessing. Report: 1. Whether any HUMAN is present. 2. Whether any PLANT is present. 3. For each plant: the species (common name and scientific name). 4. For each plant: its health, and any visible disease, pest damage, nutrient deficiency, wilting, or discolouration. 5. A short practical recommendation for the grower. Confidence values are integers 0-100. Use a low confidence when the image is blurry, dark, or the subject is far away or partly hidden. If there is no plant in the frame, return an empty plants list. If you cannot identify the species, set species_common to "Unknown" and give a low species_confidence.""" RESPONSE_SCHEMA = { "type": "object", "properties": { "human_present": {"type": "boolean"}, "human_count": {"type": "integer"}, "human_confidence": {"type": "integer"}, "human_description": {"type": "string"}, "plant_present": {"type": "boolean"}, "plants": { "type": "array", "items": { "type": "object", "properties": { "species_common": {"type": "string"}, "species_scientific": {"type": "string"}, "species_confidence": {"type": "integer"}, "health_status": { "type": "string", "enum": ["healthy", "mild_issue", "diseased", "severe", "unknown"] }, "disease_name": {"type": "string"}, "disease_confidence": {"type": "integer"}, "symptoms": {"type": "string"}, "recommendation": {"type": "string"}, }, "required": ["species_common", "species_confidence", "health_status"], }, }, "scene_summary": {"type": "string"}, "alert_worthy": {"type": "boolean"}, }, "required": ["human_present", "plant_present", "scene_summary"], } # ============================================================ # STATE # ============================================================ available = False last_error = "Not initialised" active_model = None last_analysis = None last_analysis_at = None last_analysis_image = None _calls_today = 0 _budget_date = None _last_call_at = 0.0 _last_person_alert = 0.0 _last_disease_alert = 0.0 _prev_gray = None _last_motion_at = None _current_interval = ANALYSIS_INTERVAL_SECONDS _consecutive_ok = 0 _rate_limited_until = 0.0 _health_scans_done = set() _email_queue = queue.Queue(maxsize=20) _email_log = [] _call_log = [] _lock = threading.Lock() def _log_call(ok, detail=""): _call_log.append({ "time": datetime.now().strftime("%H:%M:%S"), "ok": ok, "detail": detail[:160], }) del _call_log[:-15] def _log_email(ok, subject, detail=""): _email_log.append({ "time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), "ok": ok, "subject": subject, "detail": detail, }) del _email_log[:-15] print(("EMAIL SENT: " if ok else "EMAIL FAILED: ") + subject + ((" - " + detail) if detail else "")) # ============================================================ # BUDGET PERSISTENCE # ============================================================ def _load_budget(): """Survive restarts so the daily cap cannot be bypassed by rebooting.""" global _calls_today, _budget_date today = date.today().isoformat() try: with open(STATE_FILE) as f: data = json.load(f) if data.get("date") == today: _calls_today = int(data.get("calls", 0)) _budget_date = today return except Exception: pass _calls_today = 0 _budget_date = today def _save_budget(): try: os.makedirs(BASE_DIR, exist_ok=True) with open(STATE_FILE, "w") as f: json.dump({"date": _budget_date, "calls": _calls_today}, f) except Exception: pass def _roll_day_if_needed(): global _calls_today, _budget_date, _health_scans_done today = date.today().isoformat() if _budget_date != today: _budget_date = today _calls_today = 0 _health_scans_done = set() _save_budget() def budget_remaining(): _roll_day_if_needed() if DAILY_CALL_BUDGET <= 0: return None # no cap configured return max(0, DAILY_CALL_BUDGET - _calls_today) def _apply_backoff(reason): """Slow down after a rate-limit rejection instead of hammering.""" global _current_interval, _consecutive_ok _consecutive_ok = 0 if not AUTO_BACKOFF: return old = _current_interval _current_interval = min(MAX_BACKOFF_SECONDS, max(1.0, _current_interval) * BACKOFF_MULTIPLIER) if _current_interval != old: print("Gemini backoff: %gs -> %gs (%s)" % (round(old, 1), round(_current_interval, 1), reason)) def _recover_interval(): """Ease back toward the configured interval after sustained success.""" global _current_interval, _consecutive_ok _consecutive_ok += 1 if not AUTO_BACKOFF: return if _consecutive_ok >= 5 and _current_interval > ANALYSIS_INTERVAL_SECONDS: old = _current_interval _current_interval = max(ANALYSIS_INTERVAL_SECONDS, _current_interval / BACKOFF_MULTIPLIER) _consecutive_ok = 0 print("Gemini recovering: %gs -> %gs" % (round(old, 1), round(_current_interval, 1))) # ============================================================ # GEMINI CALL # ============================================================ def _post_json(url, payload): body = json.dumps(payload).encode("utf-8") req = urllib.request.Request( url, data=body, headers={"Content-Type": "application/json"}, method="POST", ) ctx = ssl.create_default_context() with urllib.request.urlopen(req, timeout=REQUEST_TIMEOUT, context=ctx) as resp: return json.loads(resp.read().decode("utf-8")) def _encode_frame(frame_bgr, max_width=640, quality=70): """JPEG-encode and base64 a frame. Smaller = fewer tokens = faster.""" if cv2 is None: raise RuntimeError("OpenCV not installed") h, w = frame_bgr.shape[:2] if w > max_width: scale = max_width / float(w) frame_bgr = cv2.resize(frame_bgr, (max_width, int(h * scale))) ok, buf = cv2.imencode(".jpg", frame_bgr, [cv2.IMWRITE_JPEG_QUALITY, quality]) if not ok: raise RuntimeError("JPEG encode failed") return base64.b64encode(buf.tobytes()).decode("ascii") def analyse_frame(frame_bgr, force=False): """Send one frame to Gemini. Returns a parsed dict, or None.""" global _calls_today, _last_call_at, active_model global available, last_error, last_analysis, last_analysis_at global _rate_limited_until if not GEMINI_ENABLED: last_error = "GEMINI_API_KEY not set" return None if frame_bgr is None: return None _roll_day_if_needed() now = time.time() if not force: if DAILY_CALL_BUDGET > 0 and _calls_today >= DAILY_CALL_BUDGET: last_error = ("Daily Gemini budget spent (%d). Resets at midnight." % DAILY_CALL_BUDGET) return None if now < _rate_limited_until: return None if now - _last_call_at < _current_interval: return None try: b64 = _encode_frame(frame_bgr) except Exception as e: last_error = "Encode failed: " + str(e) return None payload = { "contents": [{ "parts": [ {"inline_data": {"mime_type": "image/jpeg", "data": b64}}, {"text": ANALYSIS_PROMPT}, ] }], "generationConfig": { "responseMimeType": "application/json", "responseSchema": RESPONSE_SCHEMA, "temperature": 0.1, }, } models = ([active_model] if active_model else []) + \ [m for m in GEMINI_MODELS if m != active_model] for model in models: url = "%s/%s:generateContent?key=%s" % (API_ROOT, model, GEMINI_API_KEY) try: with _lock: _calls_today += 1 _last_call_at = time.time() _save_budget() raw = _post_json(url, payload) text = (raw["candidates"][0]["content"]["parts"][0]["text"]) result = json.loads(text) active_model = model available = True last_error = "" last_analysis = result last_analysis_at = datetime.now().strftime("%Y-%m-%d %H:%M:%S") _recover_interval() _log_call(True, model) return result except urllib.error.HTTPError as e: detail = "" try: detail = e.read().decode("utf-8", errors="ignore")[:200] except Exception: pass msg = "%s HTTP %s %s" % (model, e.code, detail) _log_call(False, msg) if e.code in (400, 404): continue # bad/retired model - try the next one if e.code == 429: _apply_backoff("HTTP 429") _rate_limited_until = time.time() + _current_interval last_error = ("Rate limited / quota exceeded. Slowed to " "%gs between calls. %s" % (round(_current_interval, 1), detail[:120])) available = False return None if e.code in (401, 403): last_error = "API key rejected: " + detail available = False return None last_error = msg except Exception as e: _log_call(False, "%s %s" % (model, e)) last_error = "%s: %s" % (model, e) available = False return None # ============================================================ # MOTION GATE - free, local, keeps the API bill at zero # ============================================================ def motion_score(frame_bgr): """Fraction of pixels that changed since the previous frame.""" global _prev_gray if cv2 is None or np is None or frame_bgr is None: return 0.0 try: small = cv2.resize(frame_bgr, (160, 120)) gray = cv2.cvtColor(small, cv2.COLOR_BGR2GRAY) gray = cv2.GaussianBlur(gray, (5, 5), 0) if _prev_gray is None: _prev_gray = gray return 0.0 diff = cv2.absdiff(_prev_gray, gray) _prev_gray = gray _, thresh = cv2.threshold(diff, 25, 255, cv2.THRESH_BINARY) return float(np.count_nonzero(thresh)) / thresh.size except Exception: return 0.0 # ============================================================ # SNAPSHOTS # ============================================================ def save_snapshot(frame_bgr, prefix): try: os.makedirs(SNAPSHOT_DIR, exist_ok=True) name = "%s_%s.jpg" % (prefix, datetime.now().strftime("%Y%m%d_%H%M%S")) path = os.path.join(SNAPSHOT_DIR, name) cv2.imwrite(path, frame_bgr, [cv2.IMWRITE_JPEG_QUALITY, 80]) _prune_snapshots() return path except Exception as e: print("Snapshot error:", e) return None def _prune_snapshots(keep_days=7, max_files=200): try: files = [os.path.join(SNAPSHOT_DIR, f) for f in os.listdir(SNAPSHOT_DIR) if f.endswith(".jpg")] files.sort(key=os.path.getmtime, reverse=True) cutoff = time.time() - keep_days * 86400 for i, f in enumerate(files): if i >= max_files or os.path.getmtime(f) < cutoff: os.remove(f) except Exception: pass def annotate(frame_bgr, result): """Overlay Gemini's findings on the frame.""" if frame_bgr is None or cv2 is None: return frame_bgr out = frame_bgr.copy() if not result: return out lines = [] if result.get("human_present"): lines.append(("HUMAN %d%%" % result.get("human_confidence", 0), (0, 0, 220))) for p in result.get("plants", []) or []: name = p.get("species_common", "plant") conf = p.get("species_confidence", 0) health = p.get("health_status", "unknown") colour = (60, 180, 75) if health == "healthy" else (0, 140, 255) if health in ("diseased", "severe"): colour = (0, 0, 220) txt = "%s %d%%" % (name, conf) if health not in ("healthy", "unknown"): txt += " - " + (p.get("disease_name") or health) lines.append((txt, colour)) y = 26 for txt, colour in lines[:6]: cv2.putText(out, txt, (10, y), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 0, 0), 4) cv2.putText(out, txt, (10, y), cv2.FONT_HERSHEY_SIMPLEX, 0.6, colour, 2) y += 26 return out # ============================================================ # EMAIL # ============================================================ def _send_email_now(subject, body, image_path=None): if not EMAIL_ENABLED: _log_email(False, subject, "email not configured") return False msg = EmailMessage() msg["Subject"] = subject msg["From"] = SMTP_USER msg["To"] = ALERT_TO msg.set_content(body) if image_path and os.path.exists(image_path): try: with open(image_path, "rb") as f: msg.add_attachment(f.read(), maintype="image", subtype="jpeg", filename=os.path.basename(image_path)) except Exception as e: print("Attach error:", e) try: ctx = ssl.create_default_context() with smtplib.SMTP(SMTP_HOST, SMTP_PORT, timeout=30) as s: s.ehlo() s.starttls(context=ctx) s.login(SMTP_USER, SMTP_PASS) s.send_message(msg) _log_email(True, subject) return True except smtplib.SMTPAuthenticationError: _log_email(False, subject, "auth rejected - Gmail needs a 16-char App Password, " "not your account password") return False except Exception as e: _log_email(False, subject, str(e)) return False def _email_worker(): while True: try: subject, body, path = _email_queue.get() _send_email_now(subject, body, path) except Exception as e: print("Email worker error:", e) finally: time.sleep(1) def queue_email(subject, body, image_path=None): try: _email_queue.put_nowait((subject, body, image_path)) return True except queue.Full: print("Email queue full, dropped:", subject) return False def send_test_email(): return queue_email( "ECO-ROVER NEXUS - test alert", "Test message from your ECO-ROVER NEXUS unit.\n\nSent: %s\n" "Gemini configured: %s\nModel: %s\n" % (datetime.now().strftime("%Y-%m-%d %H:%M:%S"), GEMINI_ENABLED, active_model or "not yet used")) # ============================================================ # REPORT FORMATTING # ============================================================ def format_report(result, sensors=None): if not result: return "No analysis available." out = [] out.append(result.get("scene_summary", "")) out.append("") if result.get("human_present"): out.append("HUMAN DETECTED") out.append(" Count: %s" % result.get("human_count", 1)) out.append(" Confidence: %s%%" % result.get("human_confidence", 0)) if result.get("human_description"): out.append(" Details: %s" % result["human_description"]) out.append("") plants = result.get("plants") or [] if plants: out.append("PLANTS (%d)" % len(plants)) for i, p in enumerate(plants, 1): out.append(" %d. %s" % (i, p.get("species_common", "Unknown"))) if p.get("species_scientific"): out.append(" Scientific: %s" % p["species_scientific"]) out.append(" Species confidence: %s%%" % p.get("species_confidence", 0)) out.append(" Health: %s" % p.get("health_status", "unknown")) if p.get("disease_name"): out.append(" Disease: %s (%s%% confidence)" % (p["disease_name"], p.get("disease_confidence", 0))) if p.get("symptoms"): out.append(" Symptoms: %s" % p["symptoms"]) if p.get("recommendation"): out.append(" Action: %s" % p["recommendation"]) out.append("") else: out.append("No plants identified in this frame.") out.append("") if sensors: out.append("SENSOR READINGS") for k, v in sensors.items(): out.append(" %s: %s" % (k, v)) out.append("") out.append("Analysed by Gemini (%s) at %s" % (active_model or "?", last_analysis_at or "?")) return "\n".join(out) # ============================================================ # ALERT DECISIONS # ============================================================ def _handle_result(result, frame_bgr, sensors, reason): global _last_person_alert, _last_disease_alert if not result: return now = time.time() # ---- Person ---- if (result.get("human_present") and result.get("human_confidence", 0) >= PERSON_MIN_CONFIDENCE and now - _last_person_alert >= PERSON_ALERT_COOLDOWN): _last_person_alert = now path = save_snapshot(annotate(frame_bgr, result), "person") queue_email( "ECO-ROVER ALERT: person detected (%d%%)" % result.get("human_confidence", 0), "A person was detected by the farm camera.\n\n" + format_report(result, sensors), path) # ---- Disease ---- sick = [p for p in (result.get("plants") or []) if p.get("health_status") in ("diseased", "severe") and p.get("disease_confidence", 0) >= DISEASE_MIN_CONFIDENCE] if sick and now - _last_disease_alert >= DISEASE_ALERT_COOLDOWN: _last_disease_alert = now path = save_snapshot(annotate(frame_bgr, result), "disease") names = ", ".join( "%s: %s" % (p.get("species_common", "plant"), p.get("disease_name", "issue")) for p in sick) queue_email("ECO-ROVER PLANT HEALTH: " + names[:80], "A plant health problem was detected.\n\n" + format_report(result, sensors), path) # ============================================================ # BACKGROUND LOOP # ============================================================ def start(get_frame, get_sensors=None): """Begin monitoring. get_frame() returns a BGR frame or None.""" global available, last_error _load_budget() threading.Thread(target=_email_worker, daemon=True).start() if not GEMINI_ENABLED: last_error = ("GEMINI_API_KEY not set - get a free key at " "aistudio.google.com/apikey") print("Gemini vision: " + last_error) else: per_day = int(86400 / ANALYSIS_INTERVAL_SECONDS) print("Gemini vision: enabled. Model order: %s" % ", ".join(GEMINI_MODELS[:2])) print(" Interval: %gs -> ~%d calls/day (%.1f/min)" % (ANALYSIS_INTERVAL_SECONDS, per_day, 60.0 / ANALYSIS_INTERVAL_SECONDS)) if DAILY_CALL_BUDGET > 0: print(" Daily cap: %d (%d used today)" % (DAILY_CALL_BUDGET, _calls_today)) else: print(" Daily cap: none (set ECO_GEMINI_DAILY_BUDGET to add one)") if per_day > 1500: print(" WARNING: the Gemini free tier is roughly 250-1500 " "calls/day.") print(" At %gs you will exhaust it in about %d " "minutes," % (ANALYSIS_INTERVAL_SECONDS, int(1500 * ANALYSIS_INTERVAL_SECONDS / 60))) print(" then auto-backoff will slow calls until " "midnight Pacific.") print(" Use a paid key, or raise ECO_GEMINI_INTERVAL.") print(" Auto-backoff: %s" % ("on" if AUTO_BACKOFF else "off")) print(" Motion gate : %s" % ("on" if USE_MOTION_GATE else "off (every interval)")) def loop(): global _last_motion_at, last_analysis_image while True: try: frame = get_frame() if frame is None: time.sleep(1.0) continue sensors = get_sensors() if get_sensors else None now = datetime.now() # Optional motion gate. Off by default so the configured # interval is honoured exactly. proceed = True if USE_MOTION_GATE: score = motion_score(frame) if score >= MOTION_THRESHOLD: _last_motion_at = now.strftime("%Y-%m-%d %H:%M:%S") proceed = True elif (now.hour in HEALTH_SCAN_HOURS and now.hour not in _health_scans_done): _health_scans_done.add(now.hour) proceed = True else: proceed = False if proceed and GEMINI_ENABLED: # analyse_frame() self-throttles to _current_interval, # so calling it often is cheap and safe. result = analyse_frame(frame) if result: last_analysis_image = annotate(frame, result) _handle_result(result, frame, sensors, "interval") except Exception as e: print("Gemini loop error:", e) # Poll faster than the interval so timing stays accurate, # but never spin the CPU. time.sleep(min(1.0, max(0.2, _current_interval / 4.0))) threading.Thread(target=loop, daemon=True).start() print("Gemini vision thread started.") def status(): _roll_day_if_needed() return { "gemini_enabled": GEMINI_ENABLED, "available": available, "active_model": active_model, "last_error": last_error, "calls_today": _calls_today, "daily_budget": DAILY_CALL_BUDGET, "budget_remaining": budget_remaining(), "configured_interval_s": ANALYSIS_INTERVAL_SECONDS, "current_interval_s": round(_current_interval, 1), "backed_off": _current_interval > ANALYSIS_INTERVAL_SECONDS, "estimated_calls_per_day": int(86400 / _current_interval) if _current_interval > 0 else None, "motion_gate": USE_MOTION_GATE, "last_analysis_at": last_analysis_at, "last_motion_at": _last_motion_at, "analysis": last_analysis, "email_configured": EMAIL_ENABLED, "alert_to": ALERT_TO if EMAIL_ENABLED else "", "recent_calls": _call_log[-8:], "recent_emails": _email_log[-8:], } def analyse_now(frame_bgr, sensors=None): """Manual 'analyse this instant' for the dashboard button.""" result = analyse_frame(frame_bgr, force=True) if result: global last_analysis_image last_analysis_image = annotate(frame_bgr, result) _handle_result(result, frame_bgr, sensors, "manual") return result