Add vision and firecrawl tools

Author: srilaasya

agentstack/_tools/agentql/__init__.py

@@ -17,32 +17,32 @@ def query_data(url: str, query: Optional[str], prompt: Optional[str]) -> dict:
 
     AgentQL query to scrape the url.
 
-Here is a guide on AgentQL query syntax:
+    Here is a guide on AgentQL query syntax:
 
-Enclose all AgentQL query terms within curly braces `{}`. The following query structure isn't valid because the term "social\_media\_links" is wrongly enclosed within parenthesis `()`.
+    Enclose all AgentQL query terms within curly braces `{}`. The following query structure isn't valid because the term "social/media/links" is wrongly enclosed within parenthesis `()`.
 
-```
-( # Should be {
-    social_media_links(The icons that lead to Facebook, Snapchat, etc.)[]
-) # Should be }
-```
+    ```
+    ( # Should be {
+        social_media_links(The icons that lead to Facebook, Snapchat, etc.)[]
+    ) # Should be }
+    ```
 
-The following query is also invalid since its missing the curly braces `{}`
+    The following query is also invalid since its missing the curly braces `{}`
 
-```
-# should include {
-social_media_links(The icons that lead to Facebook, Snapchat, etc.)[]
-# should include }
-```
+    ```
+    # should include {
+    social_media_links(The icons that lead to Facebook, Snapchat, etc.)[]
+    # should include }
+    ```
 
-You can't include new lines in your semantic context. The following query structure isn't valid because the semantic context isn't contained within one line.
+    You can't include new lines in your semantic context. The following query structure isn't valid because the semantic context isn't contained within one line.
 
-```
-{
-    social_media_links(The icons that lead
-        to Facebook, Snapchat, etc.)[]
-}
-```
+    ```
+    {
+        social_media_links(The icons that lead
+            to Facebook, Snapchat, etc.)[]
+    }
+    ```
     """
     payload = {
         "url": url,

agentstack/_tools/vision/__init__.py

@@ -1,51 +1,68 @@
-"""Vision tool for analyzing images using OpenAI's Vision API."""
+"""Vision tool for analyzing images using OpenAI's Vision API and Claude."""
 
 import base64
-from typing import Optional
+from typing import Optional, Literal
 import requests
 from openai import OpenAI
+import anthropic
 
 __all__ = ["analyze_image"]
 
 
-def analyze_image(image_path_url: str) -> str:
+def analyze_image(image_path_url: str, model: Literal["openai", "claude"] = "openai") -> str:
     """
-    Analyze an image using OpenAI's Vision API.
+    Analyze an image using either OpenAI's Vision API or Claude.
 
     Args:
         image_path_url: Local path or URL to the image
+        model: Which model to use ("openai" or "claude"). Defaults to "openai"
 
     Returns:
         str: Description of the image contents
     """
-    client = OpenAI()
-
     if not image_path_url:
         return "Image Path or URL is required."
 
-    if "http" in image_path_url:
-        return _analyze_web_image(client, image_path_url)
-    return _analyze_local_image(client, image_path_url)
+    if model == "openai":
+        client = OpenAI()
+        if "http" in image_path_url:
+            return _analyze_web_image_openai(client, image_path_url)
+        return _analyze_local_image_openai(client, image_path_url)
+    elif model == "claude":
+        client = Anthropic()
+        if "http" in image_path_url:
+            return _analyze_web_image_claude(client, image_path_url)
+        return _analyze_local_image_claude(client, image_path_url)
+    else:
+        raise ValueError("Model must be either 'openai' or 'claude'")
 
 
-def _analyze_web_image(client: OpenAI, image_path_url: str) -> str:
-    response = client.chat.completions.create(
-        model="gpt-4-vision-preview",
-        messages=[
-            {
+def _analyze_web_image_openai(client: OpenAI, image_url: str) -> str:
+    """Analyze a web-hosted image using OpenAI's Vision API."""
+    try:
+        response = client.chat.completions.create(
+            model="gpt-4-vision-preview",
+            messages=[{
                 "role": "user",
                 "content": [
-                    {"type": "text", "text": "What's in this image?"},
-                    {"type": "image_url", "image_url": {"url": image_path_url}},
-                ],
-            }
-        ],
-        max_tokens=300,
-    )
-    return response.choices[0].message.content  # type: ignore[return-value]
+                    {
+                        "type": "text",
+                        "text": "What's in this image?"
+                    },
+                    {
+                        "type": "image_url",
+                        "image_url": {"url": image_url}
+                    }
+                ]
+            }],
+            max_tokens=300
+        )
+        return response.choices[0].message.content or "No description available"
+    except Exception as e:
+        return f"Error analyzing image: {str(e)}"
 
 
-def _analyze_local_image(client: OpenAI, image_path: str) -> str:
+def _analyze_local_image_openai(client: OpenAI, image_path: str) -> str:
     base64_image = _encode_image(image_path)
     headers = {"Content-Type": "application/json", "Authorization": f"Bearer {client.api_key}"}
     payload = {
@@ -65,6 +82,46 @@ def _analyze_local_image(client: OpenAI, image_path: str) -> str:
     return response.json()["choices"][0]["message"]["content"]
 
 
+def _analyze_web_image_claude(client: Anthropic, image_path_url: str) -> str:
+    response = client.messages.create(
+        model="claude-3-opus-20240229",
+        max_tokens=300,
+        messages=[{
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "What's in this image?"},
+                {"type": "image", "source": {"type": "url", "url": image_path_url}}
+            ]
+        }]
+    )
+    return response.content[0].text
+
+
+def _analyze_local_image_claude(client: Anthropic, image_path: str) -> str:
+    with open(image_path, "rb") as image_file:
+        media_data = image_file.read()
+    
+    response = client.messages.create(
+        model="claude-3-opus-20240229",
+        max_tokens=300,
+        messages=[{
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "What's in this image?"},
+                {
+                    "type": "image",
+                    "source": {
+                        "type": "base64",
+                        "media_type": "image/jpeg",
+                        "data": base64.b64encode(media_data).decode()
+                    }
+                }
+            ]
+        }]
+    )
+    return response.content[0].text
+
+
 def _encode_image(image_path: str) -> str:
     with open(image_path, "rb") as image_file:
-        return base64.b64encode(image_file.read()).decode("utf-8")
+        return base64.b64encode(image_file.read()).decode("utf-8")

agentstack/_tools/vision/config.json

@@ -6,7 +6,10 @@
   },
   "dependencies": [
     "openai>=1.0.0",
-    "requests>=2.31.0"
+    "requests>=2.31.0",
+    "anthropic"
   ],
-  "tools": ["analyze_image"]
-}
+  "tools": [
+    "analyze_image"
+  ]
+}