From 3f9cf5eb066c4b0813fa919e9a150fd5a45e680f Mon Sep 17 00:00:00 2001 From: Hayden <154503486+groupthinking@users.noreply.github.com> Date: Wed, 2 Sep 2026 02:45:01 -0500 Subject: [PATCH 1/4] feat: add Gemini agentic video adapter --- src/integration/gemini_agentic_video.py | 113 ++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 src/integration/gemini_agentic_video.py diff --git a/src/integration/gemini_agentic_video.py b/src/integration/gemini_agentic_video.py new file mode 100644 index 000000000..277fb0005 --- /dev/null +++ b/src/integration/gemini_agentic_video.py @@ -0,0 +1,113 @@ +"""Gemini agentic video-understanding adapter. + +This module isolates the Gemini Interactions API from EventRelay's existing +``generateContent`` integration. It enables targeted, server-side inspection +of transcripts, frames, and audio without changing the production path until +benchmark evidence supports promotion. +""" + +from __future__ import annotations + +import asyncio +import os +from dataclasses import dataclass +from typing import Any, Literal, Sequence + +ProcessingMode = Literal["agentic", "static"] + + +@dataclass(frozen=True) +class VideoInput: + """One video reference and its independently selected processing mode.""" + + uri: str + processing: ProcessingMode = "agentic" + mime_type: str | None = None + + +@dataclass(frozen=True) +class AgenticVideoReceipt: + """Stable execution receipt retained by EventRelay after analysis.""" + + output_text: str + total_tokens: int | None + model: str + sources: tuple[str, ...] + processing_modes: tuple[ProcessingMode, ...] + + +class GeminiAgenticVideoService: + """Run Gemini's Think -> Act -> Observe video-analysis loop.""" + + DEFAULT_MODEL = "gemini-3.7-flash" + + def __init__( + self, + api_key: str | None = None, + *, + client: Any | None = None, + model: str | None = None, + ) -> None: + self.model = model or os.getenv( + "GEMINI_AGENTIC_VIDEO_MODEL", self.DEFAULT_MODEL + ) + if client is not None: + self._client = client + return + + from google import genai + + resolved_key = api_key or os.getenv("GEMINI_API_KEY") + self._client = ( + genai.Client(api_key=resolved_key) if resolved_key else genai.Client() + ) + + @staticmethod + def build_input(videos: Sequence[VideoInput], prompt: str) -> list[dict[str, str]]: + """Build the documented Interactions API input without materializing media.""" + if not videos: + raise ValueError("At least one video reference is required") + if not prompt.strip(): + raise ValueError("A non-empty analysis prompt is required") + + items: list[dict[str, str]] = [] + for video in videos: + if not video.uri.strip(): + raise ValueError("Video URI must not be empty") + item = { + "type": "video", + "uri": video.uri, + "processing": video.processing, + } + if video.mime_type: + item["mime_type"] = video.mime_type + items.append(item) + items.append({"type": "text", "text": prompt}) + return items + + async def analyze( + self, + videos: Sequence[VideoInput], + prompt: str, + *, + model: str | None = None, + ) -> AgenticVideoReceipt: + """Analyze referenced media and return a durable, comparable receipt.""" + selected_model = model or self.model + request_input = self.build_input(videos, prompt) + response = await asyncio.to_thread( + self._client.interactions.create, + model=selected_model, + input=request_input, + ) + + usage = getattr(response, "usage", None) + total_tokens = getattr(usage, "total_tokens", None) + return AgenticVideoReceipt( + output_text=str(getattr(response, "output_text", "")), + total_tokens=int(total_tokens) if total_tokens is not None else None, + model=selected_model, + sources=tuple(video.uri for video in videos), + processing_modes=tuple(video.processing for video in videos), + ) + From 18656dca37d21d640a8eb316f3c8692c1ac2cffb Mon Sep 17 00:00:00 2001 From: Hayden <154503486+groupthinking@users.noreply.github.com> Date: Wed, 2 Sep 2026 02:45:02 -0500 Subject: [PATCH 2/4] test: cover Gemini agentic video adapter --- tests/unit/test_gemini_agentic_video.py | 74 +++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 tests/unit/test_gemini_agentic_video.py diff --git a/tests/unit/test_gemini_agentic_video.py b/tests/unit/test_gemini_agentic_video.py new file mode 100644 index 000000000..3859366ac --- /dev/null +++ b/tests/unit/test_gemini_agentic_video.py @@ -0,0 +1,74 @@ +from types import SimpleNamespace + +import pytest + +from src.integration.gemini_agentic_video import ( + GeminiAgenticVideoService, + VideoInput, +) + + +class FakeInteractions: + def __init__(self) -> None: + self.request = None + + def create(self, **kwargs): + self.request = kwargs + return SimpleNamespace( + output_text="grounded result", + usage=SimpleNamespace(total_tokens=321), + ) + + +@pytest.mark.asyncio +async def test_agentic_youtube_request_returns_execution_receipt(): + interactions = FakeInteractions() + client = SimpleNamespace(interactions=interactions) + service = GeminiAgenticVideoService(client=client) + + receipt = await service.analyze( + [VideoInput("https://youtu.be/auJzb1D-fag")], + "Find the implementation steps and their timestamps.", + ) + + assert interactions.request == { + "model": "gemini-3.7-flash", + "input": [ + { + "type": "video", + "uri": "https://youtu.be/auJzb1D-fag", + "processing": "agentic", + }, + { + "type": "text", + "text": "Find the implementation steps and their timestamps.", + }, + ], + } + assert receipt.output_text == "grounded result" + assert receipt.total_tokens == 321 + assert receipt.sources == ("https://youtu.be/auJzb1D-fag",) + assert receipt.processing_modes == ("agentic",) + + +def test_mixed_mode_keeps_each_video_processing_policy(): + request_input = GeminiAgenticVideoService.build_input( + [ + VideoInput("gs://bucket/reference.mp4", "agentic", "video/mp4"), + VideoInput("gs://bucket/clip.mp4", "static", "video/mp4"), + ], + "Locate the clip in the reference recording.", + ) + + assert request_input[0]["processing"] == "agentic" + assert request_input[1]["processing"] == "static" + assert request_input[0]["mime_type"] == "video/mp4" + + +@pytest.mark.parametrize( + ("videos", "prompt"), + [([], "question"), ([VideoInput("")], "question"), ([VideoInput("x")], " ")], +) +def test_invalid_requests_fail_before_calling_provider(videos, prompt): + with pytest.raises(ValueError): + GeminiAgenticVideoService.build_input(videos, prompt) From 18c876a9141dc90c2e118f9b8fff82656f4aca4a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 9 Sep 2026 19:11:06 +0000 Subject: [PATCH 3/4] fix: validate agentic video inputs and receipts Co-authored-by: groupthinking <154503486+groupthinking@users.noreply.github.com> --- src/integration/gemini_agentic_video.py | 42 +++++++++++++++++++++++-- tests/unit/test_gemini_agentic_video.py | 21 +++++++++++-- 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/src/integration/gemini_agentic_video.py b/src/integration/gemini_agentic_video.py index 277fb0005..0d33faa9c 100644 --- a/src/integration/gemini_agentic_video.py +++ b/src/integration/gemini_agentic_video.py @@ -12,6 +12,7 @@ import os from dataclasses import dataclass from typing import Any, Literal, Sequence +from urllib.parse import urlsplit ProcessingMode = Literal["agentic", "static"] @@ -72,8 +73,8 @@ def build_input(videos: Sequence[VideoInput], prompt: str) -> list[dict[str, str items: list[dict[str, str]] = [] for video in videos: - if not video.uri.strip(): - raise ValueError("Video URI must not be empty") + if not GeminiAgenticVideoService._is_supported_uri(video.uri): + raise ValueError("Video URI must be a supported YouTube or file URI") item = { "type": "video", "uri": video.uri, @@ -85,6 +86,31 @@ def build_input(videos: Sequence[VideoInput], prompt: str) -> list[dict[str, str items.append({"type": "text", "text": prompt}) return items + @staticmethod + def _is_supported_uri(uri: str) -> bool: + if not uri.strip() or uri != uri.strip(): + return False + try: + parsed = urlsplit(uri) + hostname = (parsed.hostname or "").lower() + except ValueError: + return False + + if parsed.scheme == "https": + return ( + bool(parsed.path) + and ( + hostname == "youtu.be" + or hostname == "youtube.com" + or hostname.endswith(".youtube.com") + ) + ) + if parsed.scheme == "gs": + return bool(parsed.netloc and parsed.path) + if parsed.scheme == "file": + return bool(parsed.path) + return False + async def analyze( self, videos: Sequence[VideoInput], @@ -104,10 +130,20 @@ async def analyze( usage = getattr(response, "usage", None) total_tokens = getattr(usage, "total_tokens", None) return AgenticVideoReceipt( - output_text=str(getattr(response, "output_text", "")), + output_text=self._response_text(response), total_tokens=int(total_tokens) if total_tokens is not None else None, model=selected_model, sources=tuple(video.uri for video in videos), processing_modes=tuple(video.processing for video in videos), ) + @staticmethod + def _response_text(response: Any) -> str: + output_text = getattr(response, "output_text", None) + if output_text is not None: + return str(output_text) + return "".join( + str(getattr(output, "text", "")) + for output in (getattr(response, "outputs", None) or []) + if getattr(output, "text", None) is not None + ) diff --git a/tests/unit/test_gemini_agentic_video.py b/tests/unit/test_gemini_agentic_video.py index 3859366ac..68ce262ca 100644 --- a/tests/unit/test_gemini_agentic_video.py +++ b/tests/unit/test_gemini_agentic_video.py @@ -1,6 +1,8 @@ +from datetime import datetime, timezone from types import SimpleNamespace import pytest +from google.genai._interactions.types import Interaction, TextContent, Usage from src.integration.gemini_agentic_video import ( GeminiAgenticVideoService, @@ -14,9 +16,13 @@ def __init__(self) -> None: def create(self, **kwargs): self.request = kwargs - return SimpleNamespace( - output_text="grounded result", - usage=SimpleNamespace(total_tokens=321), + return Interaction( + id="interaction-123", + created=datetime.now(timezone.utc), + status="completed", + updated=datetime.now(timezone.utc), + outputs=[TextContent(type="text", text="grounded result")], + usage=Usage(total_tokens=321), ) @@ -65,6 +71,15 @@ def test_mixed_mode_keeps_each_video_processing_policy(): assert request_input[0]["mime_type"] == "video/mp4" +@pytest.mark.parametrize( + "uri", + ["x", "https://example.com/video.mp4", "ftp://youtu.be/auJzb1D-fag"], +) +def test_malformed_video_uri_fails_before_calling_provider(uri): + with pytest.raises(ValueError): + GeminiAgenticVideoService.build_input([VideoInput(uri)], "question") + + @pytest.mark.parametrize( ("videos", "prompt"), [([], "question"), ([VideoInput("")], "question"), ([VideoInput("x")], " ")], From d0215572b2a33493fa31d9ebdb81bbf84c04876a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 9 Sep 2026 20:14:10 +0000 Subject: [PATCH 4/4] fix: require Gemini SDK for agentic video Co-authored-by: groupthinking <154503486+groupthinking@users.noreply.github.com> --- pyproject.toml | 2 +- requirements.txt | 2 +- tests/unit/test_gemini_agentic_video.py | 14 +++++++++----- uv.lock | 8 ++++---- 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1f0860ef5..1c12fb2e8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,7 +66,7 @@ dependencies = [ "PyYAML>=6.0.0", "requests>=2.31.0", "Pillow>=10.0.0", - "google-genai>=1.0.0", + "google-genai>=2.21.0", "google-generativeai>=0.3.0", "opentelemetry-distro>=0.40b0", "opentelemetry-exporter-otlp>=1.20.0", diff --git a/requirements.txt b/requirements.txt index 1e7d37739..f88340256 100644 --- a/requirements.txt +++ b/requirements.txt @@ -55,7 +55,7 @@ psutil>=5.9.0 cachetools>=5.0.0 # Google AI & Cloud -google-genai>=1.0.0 +google-genai>=2.21.0 google-generativeai>=0.3.0 google-cloud-aiplatform>=1.133.0 google-cloud-vision>=3.7.0 diff --git a/tests/unit/test_gemini_agentic_video.py b/tests/unit/test_gemini_agentic_video.py index 68ce262ca..3acea2b98 100644 --- a/tests/unit/test_gemini_agentic_video.py +++ b/tests/unit/test_gemini_agentic_video.py @@ -1,8 +1,10 @@ -from datetime import datetime, timezone from types import SimpleNamespace import pytest -from google.genai._interactions.types import Interaction, TextContent, Usage +from google.genai._gaos.types.interactions.interaction import Interaction +from google.genai._gaos.types.interactions.modeloutputstep import ModelOutputStep +from google.genai._gaos.types.interactions.textcontent import TextContent +from google.genai._gaos.types.interactions.usage import Usage from src.integration.gemini_agentic_video import ( GeminiAgenticVideoService, @@ -18,10 +20,12 @@ def create(self, **kwargs): self.request = kwargs return Interaction( id="interaction-123", - created=datetime.now(timezone.utc), + created="2026-09-09T20:00:00Z", status="completed", - updated=datetime.now(timezone.utc), - outputs=[TextContent(type="text", text="grounded result")], + updated="2026-09-09T20:00:01Z", + steps=[ + ModelOutputStep(content=[TextContent(text="grounded result")]) + ], usage=Usage(total_tokens=321), ) diff --git a/uv.lock b/uv.lock index 57d3b3af5..d2fd63c01 100644 --- a/uv.lock +++ b/uv.lock @@ -2416,7 +2416,7 @@ wheels = [ [[package]] name = "google-genai" -version = "1.75.0" +version = "2.21.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -2430,9 +2430,9 @@ dependencies = [ { name = "typing-extensions" }, { name = "websockets" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9d/59/3ed61240ef20b3ae6ed54e82c6f8b6d1f194947bc6679679dd6cdb037594/google_genai-1.75.0.tar.gz", hash = "sha256:56bac3991b311c93f980c0a2abcd287b672146905df1fbd71c92ed633d5a07cf", size = 539039, upload-time = "2026-05-04T22:48:54.857Z" } +sdist = { url = "https://files.pythonhosted.org/packages/60/a7/a45f64f22ab9302b55fcbeb32acb6f313690a7748629b01e451aad1817a3/google_genai-2.21.0.tar.gz", hash = "sha256:0ecc11c6a5b9f5e3cc58e77ae5fead00c6719f8a1b2b654b803f514a9a6b64c0", size = 677301, upload-time = "2026-08-31T21:49:14.508781Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2d/b6/552d40e96da22921eb1fead7c14b00b5b5473a20e45959488660fab35ee2/google_genai-1.75.0-py3-none-any.whl", hash = "sha256:8dc4c096e7d6288c3087f6893f582fe52468932464781edb8193bd92b9fefb2c", size = 793726, upload-time = "2026-05-04T22:48:53.033Z" }, + { url = "https://files.pythonhosted.org/packages/fb/d7/c2419dd5fedd5803ce09810e4e39561a0a13a960b3f48d2581c12e42af3e/google_genai-2.21.0-py3-none-any.whl", hash = "sha256:36b575034be46a03acd603a852e22a6359f2cdd6b26bb1d65d9b7e0cc7ab3648", size = 1080223, upload-time = "2026-08-31T21:49:12.699394Z" }, ] [[package]] @@ -8737,7 +8737,7 @@ requires-dist = [ { name = "google-cloud-tasks", marker = "extra == 'cloud'", specifier = ">=2.14.0" }, { name = "google-cloud-videointelligence", marker = "extra == 'youtube'", specifier = ">=2.13.0" }, { name = "google-cloud-vision", marker = "extra == 'youtube'", specifier = ">=3.7.0" }, - { name = "google-genai", specifier = ">=1.0.0" }, + { name = "google-genai", specifier = ">=2.21.0" }, { name = "google-generativeai", specifier = ">=0.3.0" }, { name = "google-generativeai", marker = "extra == 'ml'", specifier = ">=0.3.0" }, { name = "httpx", specifier = ">=0.25.0" },