fixed workflow and added tests for compile_llms_txt.py, converted connection to a private function

Author: srilaasya

.github/workflows/compile-llms-txt.yml

@@ -19,7 +19,7 @@ jobs:
       - name: Compile llms.txt
         run: |
           cd docs
-          python compile_llms.py
+          python compile_llms_txt.py
       - name: Commit and push changes
         run: |
           git config --local user.email "action@github.com"

agentstack/_tools/py_sql/__init__.py

@@ -2,7 +2,7 @@
 import psycopg2
 from typing import Dict, Any
 
-def get_connection():
+def _get_connection():
     """Get PostgreSQL database connection"""
     return psycopg2.connect(
         dbname=os.getenv('POSTGRES_DB'),
@@ -18,7 +18,7 @@ def get_schema() -> Dict[str, Any]:
     Returns a dictionary containing the database schema.
     """
     try:
-        conn = get_connection()
+        conn = _get_connection()
         cursor = conn.cursor()
 
         # Query to get all tables in the current schema
@@ -63,7 +63,7 @@ def execute_query(query: str) -> list:
         List of query results
     """
     try:
-        conn = get_connection()
+        conn = _get_connection()
         cursor = conn.cursor()
 
         # Execute the query

agentstack/_tools/py_sql/config.json

@@ -3,28 +3,11 @@
     "url": "https://pypi.org/project/psycopg2/",
     "category": "database",
     "env": {
-        "POSTGRES_DB": {
-            "description": "PostgreSQL database name",
-            "required": true
-        },
-        "POSTGRES_USER": {
-            "description": "PostgreSQL username",
-            "required": true
-        },
-        "POSTGRES_PASSWORD": {
-            "description": "PostgreSQL password",
-            "required": true
-        },
-        "POSTGRES_HOST": {
-            "description": "PostgreSQL host address",
-            "required": true,
-            "default": "localhost"
-        },
-        "POSTGRES_PORT": {
-            "description": "PostgreSQL port number",
-            "required": true,
-            "default": "5432"
-        }
+        "POSTGRES_DB": "...",
+        "POSTGRES_USER": "...",
+        "POSTGRES_PASSWORD": "...",
+        "POSTGRES_HOST": "...",
+        "POSTGRES_PORT": "..."
     },
     "dependencies": [
         "psycopg2-binary>=2.9.9"

agentstack/_tools/py_sql/test.db

@@ -1,24 +1,26 @@
 import os
+from pathlib import Path
 
 def compile_llms_txt():
-    # Get the docs directory path (where this script is located)
-    docs_dir = os.path.dirname(os.path.abspath(__file__))
+    # Get the current working directory
+    current_dir = Path(os.getcwd())
     content = ''
+    
     # Define names of directories and files to exclude
     excluded_names = {'tool'}
 
-    # Change to docs directory
-    os.chdir(docs_dir)
-    
     for root, _, files in os.walk('.'):
         # Get the last part of the current directory
         current_dir = os.path.basename(root)
         if current_dir in excluded_names:
             continue
 
         for file in files:
+            # Check if the file is an MDX file and not in excluded names
             if file.endswith('.mdx'):
-                if file in excluded_names:
+                # Extract the base name without extension for exclusion check
+                base_name = os.path.splitext(file)[0]
+                if base_name in excluded_names:
                     continue
 
                 file_path = os.path.join(root, file)
@@ -28,10 +30,9 @@ def compile_llms_txt():
                     file_content = f.read()
                 content += f"## {relative_path}\n\n{file_content}\n\n"
 
-    # Write the complete content, replacing the existing file
-    output_path = os.path.join(docs_dir, 'llms.txt')
-    with open(output_path, 'w', encoding='utf-8') as f:
-        f.write(content)
+    # Write the complete content to llms.txt in the current directory
+    output_path = Path('llms.txt')
+    output_path.write_text(content, encoding='utf-8')
 
 if __name__ == "__main__":
     compile_llms_txt()

agentstack/_tools/py_sql/test.py

@@ -0,0 +1,91 @@
+import os
+import shutil
+import tempfile
+import unittest
+from pathlib import Path
+
+import sys
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from docs.compile_llms_txt import compile_llms_txt
+
+class TestCompileLLMsTxt(unittest.TestCase):
+    def setUp(self):
+        # Create a temporary directory for test files
+        self.test_dir = tempfile.mkdtemp()
+        self.docs_dir = Path(self.test_dir)
+        
+        # Save current working directory
+        self.original_cwd = os.getcwd()
+        os.chdir(self.docs_dir)
+        
+    def tearDown(self):
+        os.chdir(self.original_cwd)
+        shutil.rmtree(self.test_dir)
+        
+    def create_test_mdx_file(self, path: str, content: str):
+        """Helper to create test MDX files"""
+        file_path = self.docs_dir / path
+        file_path.parent.mkdir(parents=True, exist_ok=True)
+        file_path.write_text(content)
+        
+    def test_basic_compilation(self):
+        """Test basic MDX file compilation"""
+        # Create test MDX files
+        self.create_test_mdx_file("test1.mdx", "Test content 1")
+        self.create_test_mdx_file("test2.mdx", "Test content 2")
+        
+        # Run compilation
+        compile_llms_txt()
+        
+        # Check output file exists and contains expected content
+        output_path = self.docs_dir / "llms.txt"
+        self.assertTrue(output_path.exists())
+        
+        content = output_path.read_text()
+        self.assertIn("## test1.mdx", content)
+        self.assertIn("Test content 1", content)
+        self.assertIn("## test2.mdx", content)
+        self.assertIn("Test content 2", content)
+        
+    def test_excluded_directories(self):
+        """Test that files in excluded directories are skipped"""
+        # Create files in both regular and excluded directories
+        self.create_test_mdx_file("regular/file.mdx", "Regular content")
+        self.create_test_mdx_file("tool/file.mdx", "Tool content")
+        
+        compile_llms_txt()
+        
+        content = (self.docs_dir / "llms.txt").read_text()
+        self.assertIn("Regular content", content)
+        self.assertNotIn("Tool content", content)
+        
+    def test_excluded_files(self):
+        """Test that excluded files are skipped"""
+        self.create_test_mdx_file("regular.mdx", "Regular content")
+        self.create_test_mdx_file("tool.mdx", "Tool content")
+        
+        compile_llms_txt()
+        
+        content = (self.docs_dir / "llms.txt").read_text()
+        self.assertIn("Regular content", content)
+        self.assertNotIn("Tool content", content)
+        
+    def test_nested_directories(self):
+        """Test compilation from nested directory structure"""
+        self.create_test_mdx_file("dir1/test1.mdx", "Content 1")
+        self.create_test_mdx_file("dir1/dir2/test2.mdx", "Content 2")
+        
+        compile_llms_txt()
+        
+        content = (self.docs_dir / "llms.txt").read_text()
+        self.assertIn("## dir1/test1.mdx", content)
+        self.assertIn("## dir1/dir2/test2.mdx", content)
+        self.assertIn("Content 1", content)
+        self.assertIn("Content 2", content)
+        
+    def test_empty_directory(self):
+        """Test compilation with no MDX files"""
+        compile_llms_txt()
+        
+        content = (self.docs_dir / "llms.txt").read_text()
+        self.assertEqual(content, "")