Add support for loading docs from a GIT repo.

Author: dpage

README.md

@@ -12,6 +12,7 @@
       - [pgEdge Document Loader Quickstart](docs/quickstart.md)
   - Using pgEdge Document Loader
       - [Using Document Loader](docs/usage.md)
+      - [Using Git Repository Sources](docs/git-sources.md)
       - [Using Custom Metadata Columns](docs/metadata.md)
       - [Updating a Document](docs/updating.md)
       - [Managing Authentication](docs/authentication.md)
@@ -35,9 +36,10 @@ The pgEdge Document Loader automatically converts documents (HTML, Markdown, reS
 **Features**
 
 - **Multiple Format Support**: HTML, Markdown, reStructuredText, and DocBook SGML/XML
+- **Git Repository Support**: Clone and process docs directly from Git repositories
 - **Automatic Conversion**: All formats converted to Markdown
 - **Metadata Extraction**: Titles, filenames, timestamps
-- **Flexible Input**: Single file, directory, or glob patterns (including `**` recursive matching)
+- **Flexible Input**: Single file, directory, glob patterns, or Git repository URL
 - **Database Flexibility**: Configurable column mappings
 - **Custom Metadata Columns**: Add fixed values to custom columns for every row
 - **Update Mode**: Update existing rows or insert new ones

cmd/pgedge-docloader/main.go

@@ -19,6 +19,7 @@ import (
 	"github.com/pgedge/pgedge-docloader/internal/config"
 	"github.com/pgedge/pgedge-docloader/internal/converter"
 	"github.com/pgedge/pgedge-docloader/internal/database"
+	"github.com/pgedge/pgedge-docloader/internal/gitsource"
 	"github.com/pgedge/pgedge-docloader/internal/processor"
 	"github.com/pgedge/pgedge-docloader/internal/types"
 )
@@ -51,10 +52,19 @@ func init() {
 	// Configuration file
 	rootCmd.Flags().StringP("config", "c", "", "Path to configuration file")
 
-	// Source configuration
+	// Source configuration - Local
 	rootCmd.Flags().StringP("source", "s", "", "Source file, directory, or glob pattern")
 	rootCmd.Flags().Bool("strip-path", false, "Strip path from filename, keeping only the base name")
 
+	// Source configuration - Git (mutually exclusive with --source)
+	rootCmd.Flags().String("git-url", "", "Git repository URL to clone and process")
+	rootCmd.Flags().String("git-branch", "", "Git branch to checkout (default: repository default)")
+	rootCmd.Flags().String("git-tag", "", "Git tag to checkout (mutually exclusive with --git-branch)")
+	rootCmd.Flags().String("git-doc-path", "", "Path within repository to process (supports glob patterns)")
+	rootCmd.Flags().String("git-clone-dir", "", "Directory to store cloned repositories (default: temp directory)")
+	rootCmd.Flags().Bool("git-keep-clone", false, "Keep cloned repository after processing")
+	rootCmd.Flags().Bool("git-skip-fetch", false, "Skip git fetch if repository already exists")
+
 	// Database connection
 	rootCmd.Flags().String("db-host", "localhost", "Database host")
 	rootCmd.Flags().Int("db-port", 5432, "Database port")
@@ -113,9 +123,30 @@ func run(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("failed to load configuration: %w", err)
 	}
 
+	// Determine source path
+	var sourcePath string
+	var gitSource *gitsource.GitSource
+
+	if cfg.GitURL != "" {
+		// Git source
+		gitSource, err = gitsource.New(cfg)
+		if err != nil {
+			return fmt.Errorf("failed to setup git source: %w", err)
+		}
+		defer func() {
+			if cleanupErr := gitSource.Cleanup(); cleanupErr != nil {
+				fmt.Fprintf(os.Stderr, "Warning: cleanup failed: %v\n", cleanupErr)
+			}
+		}()
+		sourcePath = gitSource.GetSourcePath()
+	} else {
+		// Local source
+		sourcePath = cfg.Source
+	}
+
 	// Process files
-	fmt.Printf("Processing files from: %s\n", cfg.Source)
-	documents, stats, err := processor.ProcessFiles(cfg.Source, cfg.StripPath)
+	fmt.Printf("Processing files from: %s\n", sourcePath)
+	documents, stats, err := processor.ProcessFiles(sourcePath, cfg.StripPath)
 	if err != nil {
 		return fmt.Errorf("failed to process files: %w", err)
 	}

docs/changelog.md

@@ -6,6 +6,26 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to
 [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+
+### Added
+
+- **Git repository source support**: Clone and process documentation directly
+  from Git repositories as an alternative to local files
+
+    - `--git-url` option to specify repository URL (mutually exclusive with
+      `--source`)
+    - `--git-branch` option to checkout a specific branch
+    - `--git-tag` option to checkout a specific tag (mutually exclusive with
+      `--git-branch`)
+    - `--git-doc-path` option to specify path within repository (supports glob
+      patterns)
+    - `--git-clone-dir` option for persistent clone directory
+    - `--git-keep-clone` option to preserve cloned repository after processing
+    - `--git-skip-fetch` option to skip fetch for existing clones
+    - Automatic cleanup of temporary clone directories
+    - Support for both HTTPS and SSH repository URLs
+
 ## [1.0.0-beta1] - 2025-12-15
 
 ### Changed

docs/git-sources.md

@@ -0,0 +1,206 @@
+# Using Git Repository Sources
+
+As an alternative to local files, pgEdge Document Loader can clone and process
+documentation directly from Git repositories. This is useful for:
+
+- Loading documentation from remote repositories without manual cloning
+- Processing specific branches or tags (e.g., versioned documentation)
+- Automated pipelines that fetch and load docs from source control
+
+## Git Source Options
+
+| Option           | Required | Description                                      |
+|------------------|----------|--------------------------------------------------|
+| `--git-url`      | Yes*     | Git repository URL to clone                      |
+| `--git-branch`   | No       | Branch to checkout (default: repository default) |
+| `--git-tag`      | No       | Tag to checkout (mutually exclusive with branch) |
+| `--git-doc-path` | No       | Path within repository to process                |
+| `--git-clone-dir`| No       | Directory to store cloned repositories           |
+| `--git-keep-clone`| No      | Keep cloned repository after processing          |
+| `--git-skip-fetch`| No      | Skip fetch if repository already exists          |
+
+*Either `--source` or `--git-url` is required, but not both.
+
+## Basic Usage
+
+Clone a repository and process all supported files from the root:
+
+```bash
+pgedge-docloader \
+    --git-url https://github.com/org/docs-repo.git \
+    --db-host localhost \
+    --db-name mydb \
+    --db-user myuser \
+    --db-table documents \
+    --col-doc-content content \
+    --col-file-name filename
+```
+
+## Processing a Specific Directory
+
+Use `--git-doc-path` to process files from a specific directory within the
+repository:
+
+```bash
+pgedge-docloader \
+    --git-url https://github.com/org/project.git \
+    --git-doc-path docs/api \
+    --db-host localhost \
+    --db-name mydb \
+    --db-user myuser \
+    --db-table documents \
+    --col-doc-content content
+```
+
+The `--git-doc-path` option supports glob patterns:
+
+```bash
+# Process only markdown files in the docs directory
+pgedge-docloader \
+    --git-url https://github.com/org/project.git \
+    --git-doc-path "docs/**/*.md" \
+    --config config.yml
+```
+
+## Working with Branches and Tags
+
+### Checkout a Specific Branch
+
+```bash
+pgedge-docloader \
+    --git-url https://github.com/org/docs.git \
+    --git-branch main \
+    --git-doc-path docs \
+    --config config.yml
+```
+
+### Checkout a Specific Tag
+
+Use tags for versioned documentation:
+
+```bash
+pgedge-docloader \
+    --git-url https://github.com/org/project.git \
+    --git-tag v2.0.0 \
+    --git-doc-path docs \
+    --set-column version="2.0.0" \
+    --config config.yml
+```
+
+!!! note
+
+    `--git-branch` and `--git-tag` are mutually exclusive. You cannot specify
+    both options at the same time.
+
+## Persistent Clone Directory
+
+By default, repositories are cloned to a temporary directory and removed after
+processing. For repeated runs, you can specify a persistent clone directory:
+
+```bash
+pgedge-docloader \
+    --git-url https://github.com/org/docs.git \
+    --git-clone-dir /var/cache/docloader/repos \
+    --git-keep-clone \
+    --config config.yml
+```
+
+On subsequent runs with `--git-skip-fetch`, the tool will reuse the existing
+clone without fetching updates:
+
+```bash
+pgedge-docloader \
+    --git-url https://github.com/org/docs.git \
+    --git-clone-dir /var/cache/docloader/repos \
+    --git-keep-clone \
+    --git-skip-fetch \
+    --config config.yml
+```
+
+## Configuration File Example
+
+Git source options can also be specified in a configuration file:
+
+```yaml
+# Git source configuration
+git-url: https://github.com/org/docs-repo.git
+git-branch: main
+git-doc-path: docs
+git-clone-dir: /var/cache/docloader/repos
+git-keep-clone: true
+
+# Database configuration
+db-host: localhost
+db-name: mydb
+db-user: myuser
+db-table: documents
+
+# Column mappings
+col-doc-content: content
+col-file-name: filename
+col-doc-title: title
+
+# Custom metadata
+custom-columns:
+    source: "git-repo"
+    project: "my-project"
+```
+
+Then run with:
+
+```bash
+pgedge-docloader --config config.yml
+```
+
+## Authentication
+
+### HTTPS URLs
+
+For public repositories, use the HTTPS URL directly:
+
+```bash
+--git-url https://github.com/org/public-repo.git
+```
+
+For private repositories, you can use a personal access token in the URL:
+
+```bash
+--git-url https://TOKEN@github.com/org/private-repo.git
+```
+
+Or configure Git credential helpers before running the tool.
+
+### SSH URLs
+
+For SSH authentication, ensure your SSH keys are configured:
+
+```bash
+--git-url git@github.com:org/repo.git
+```
+
+## Error Handling
+
+The tool will fail with a clear error message if:
+
+- Git is not installed on the system
+- The repository URL is invalid or inaccessible
+- The specified branch or tag does not exist
+- The `--git-doc-path` does not exist in the repository
+
+## Best Practices
+
+1. **Use tags for versioned docs**: When loading documentation for specific
+   software versions, use `--git-tag` to ensure consistency.
+
+2. **Cache clones for repeated runs**: Use `--git-clone-dir` and
+   `--git-keep-clone` to avoid re-cloning on every run.
+
+3. **Use `--git-skip-fetch` carefully**: Only skip fetching when you're sure
+   the local clone is up-to-date.
+
+4. **Set version metadata**: Use `--set-column` to add version information
+   when processing tagged releases:
+
+    ```bash
+    --git-tag v1.2.3 --set-column version="1.2.3"
+    ```

internal/config/config.go

@@ -49,9 +49,19 @@ func Load(cmd *cobra.Command) (*types.Config, error) {
 	}
 
 	// Load configuration values (CLI flags override config file)
+	// Local source configuration
 	cfg.Source = viper.GetString("source")
 	cfg.StripPath = viper.GetBool("strip-path")
 
+	// Git source configuration
+	cfg.GitURL = viper.GetString("git-url")
+	cfg.GitBranch = viper.GetString("git-branch")
+	cfg.GitTag = viper.GetString("git-tag")
+	cfg.GitDocPath = viper.GetString("git-doc-path")
+	cfg.GitCloneDir = viper.GetString("git-clone-dir")
+	cfg.GitKeepClone = viper.GetBool("git-keep-clone")
+	cfg.GitSkipFetch = viper.GetBool("git-skip-fetch")
+
 	cfg.DBHost = viper.GetString("db-host")
 	cfg.DBPort = viper.GetInt("db-port")
 	cfg.DBName = viper.GetString("db-name")
@@ -104,6 +114,7 @@ func Load(cmd *cobra.Command) (*types.Config, error) {
 	if cfg.ConfigFile != "" {
 		configDir := filepath.Dir(cfg.ConfigFile)
 		cfg.Source = resolvePath(cfg.Source, configDir)
+		cfg.GitCloneDir = resolvePath(cfg.GitCloneDir, configDir)
 		cfg.DBSSLCert = resolvePath(cfg.DBSSLCert, configDir)
 		cfg.DBSSLKey = resolvePath(cfg.DBSSLKey, configDir)
 		cfg.DBSSLRoot = resolvePath(cfg.DBSSLRoot, configDir)
@@ -198,8 +209,19 @@ func readPgPass() (string, error) {
 
 // validate validates the configuration
 func validate(cfg *types.Config) error {
-	if cfg.Source == "" {
-		return fmt.Errorf("source path is required")
+	// Source validation: either local source or git-url, but not both
+	if cfg.Source == "" && cfg.GitURL == "" {
+		return fmt.Errorf("either --source or --git-url is required")
+	}
+	if cfg.Source != "" && cfg.GitURL != "" {
+		return fmt.Errorf("--source and --git-url are mutually exclusive")
+	}
+
+	// Git-specific validation
+	if cfg.GitURL != "" {
+		if cfg.GitBranch != "" && cfg.GitTag != "" {
+			return fmt.Errorf("--git-branch and --git-tag are mutually exclusive")
+		}
 	}
 
 	if cfg.DBHost == "" {