From 84af10ebce7b7d9a848b09569e4e31560d535ab1 Mon Sep 17 00:00:00 2001 From: Divyam Talwar Date: Tue, 30 Jun 2026 06:11:00 +0530 Subject: [PATCH] Clarify LiteLLM proxy setup through registered models Issue #170 asks how to use Morphik with a LiteLLM proxy or OpenAI-compatible endpoint. The backend already accepts api_base on registered-model entries for LiteLLM/OpenAI-compatible model definitions, but the docs did not show that path clearly. Constraint: This PR must not change runtime model defaults, secret handling, or base_url alias behavior. Rejected: Supporting base_url as a registered_models alias | that is a code behavior change and needs separate tests. Confidence: high Scope-risk: narrow Directive: Keep backend registered model examples on api_base unless code adds explicit alias normalization. Tested: .venv/bin/python TOML parse for morphik.toml and morphik.docker.toml; .venv/bin/python TOML/YAML parse for DOCKER.md fenced examples; .venv/bin/python plus bash -n for DOCKER.md bash fences; commented LiteLLM proxy example, endpoint, selector-hint, embedding-dimension, and document_analysis exclusion checks; structured exact-snippet proxy docs assertions; stale/unsafe proxy docs grep; positive proxy guidance grep; LiteLLM/logging/metadata/selector-consumer code-path grep; git diff --check. Not-tested: Live LiteLLM proxy request; documentation-only change. --- DOCKER.md | 87 ++++++++++++++++++++++++++++++++++++++++----- morphik.docker.toml | 35 ++++++++++++++++++ morphik.toml | 29 +++++++++++++++ 3 files changed, 142 insertions(+), 9 deletions(-) diff --git a/DOCKER.md b/DOCKER.md index a9e02c75..360b09a5 100644 --- a/DOCKER.md +++ b/DOCKER.md @@ -54,22 +54,24 @@ The default configuration works out of the box and includes: ### 2. Configuration File (morphik.toml) -The default `morphik.toml` is configured for Docker and includes: +A Docker `morphik.toml` can configure local Ollama models with registered model keys: ```toml [api] host = "0.0.0.0" # Important: Use 0.0.0.0 for Docker port = 8000 +[registered_models] +ollama_chat = { model_name = "ollama_chat/llama3.2", api_base = "http://ollama:11434" } +ollama_embedding = { model_name = "ollama/nomic-embed-text", api_base = "http://ollama:11434" } + [completion] -provider = "ollama" -model_name = "llama3.2" -base_url = "http://ollama:11434" # Use Docker service name +model = "ollama_chat" # Reference to a key in registered_models [embedding] -provider = "ollama" -model_name = "nomic-embed-text" -base_url = "http://ollama:11434" # Use Docker service name +model = "ollama_embedding" # Reference to a key in registered_models +dimensions = 768 +similarity_metric = "cosine" [database] provider = "postgres" @@ -88,23 +90,90 @@ Create a `.env` file to customize these settings: ```bash JWT_SECRET_KEY=your-secure-key-here # Important: Change in production -OPENAI_API_KEY=sk-... # Only if using OpenAI +# OPENAI_API_KEY=your-openai-or-proxy-key # Optional: OpenAI/OpenAI-compatible proxy key HOST=0.0.0.0 # Leave as is for Docker PORT=8000 # Change if needed ``` +Keep `.env` local and out of version control when it contains real secrets. + ### 4. Custom Configuration To use your own configuration: 1. Create a custom `morphik.toml` -2. Mount it in `docker-compose.yml`: +2. Mount the same file into each service that reads Morphik configuration, including both `morphik` and `worker`: ```yaml services: morphik: volumes: - ./my-custom-morphik.toml:/app/morphik.toml + worker: + volumes: + - ./my-custom-morphik.toml:/app/morphik.toml +``` + +### 5. LiteLLM Proxy or OpenAI-Compatible Endpoints + +To use a LiteLLM proxy or another OpenAI-compatible endpoint: + +1. Define proxy-backed models under `[registered_models]`. +2. Point `[completion].model` and `[embedding].model` at those registered model keys. +3. Recreate the proxy-calling services after updating `morphik.toml` or `.env`: + +```bash +docker compose up -d --force-recreate morphik worker +``` + +After running one query and one small ingestion, you can use the logs as a smoke check for the selected model keys. This check is safe only when secrets are not embedded in registered-model config: + +```bash +docker compose logs morphik worker | grep -E "litellm_proxy_(chat|embedding)" +``` + +```toml +[registered_models] +litellm_proxy_chat = { model_name = "openai/gpt-4o-mini", api_base = "http://litellm:4000" } +litellm_proxy_embedding = { model_name = "openai/text-embedding-3-small", api_base = "http://litellm:4000" } + +[completion] +model = "litellm_proxy_chat" + +[embedding] +model = "litellm_proxy_embedding" +dimensions = 1536 +similarity_metric = "cosine" ``` +If you change `[embedding].model` or `[embedding].dimensions` on an existing deployment, keep the new embedding dimensions compatible with existing vectors or plan a re-ingestion, new collection/vector table, migration, or database reset. Recreating `morphik` and `worker` reloads config, but it does not rewrite existing pgvector data. + +Use the Docker Compose service name, such as `http://litellm:4000`, when the proxy runs in the same Compose project. Use `http://host.docker.internal:4000` only when the proxy runs on the Docker host; Linux Docker setups may require adding `extra_hosts: ["host.docker.internal:host-gateway"]` to every service that calls the proxy, at least `morphik` and `worker`: + +```yaml +services: + morphik: + extra_hosts: + - "host.docker.internal:host-gateway" + worker: + extra_hosts: + - "host.docker.internal:host-gateway" +``` + +For proxy-backed LiteLLM/OpenAI-compatible `[registered_models]` entries, put the endpoint in `api_base`. The API-key surface is separate: `/api-keys` accepts `base_url` in requests and stores/returns it as `baseUrl`; those fields do not configure backend registered models. + +Proxy-backed entries may be classified as `custom` in Morphik metadata/provider fields even when the upstream proxy exposes OpenAI-compatible models. Runtime calls still use the configured `model_name` and `api_base`. + +Credential and secret handling: + +- Set `OPENAI_API_KEY` in the shared `.env` or on every proxy-calling service, including `morphik` and `worker`, to the proxy key for general LiteLLM/OpenAI-compatible calls. +- If an authenticated embedding proxy uses a local-looking URL such as `localhost`, `127.0.0.1`, or `host.docker.internal`, also set `LITELLM_DUMMY_API_KEY` to the proxy key because Morphik forwards that value for local embedding providers. +- If your local proxy has authentication disabled, LiteLLM's OpenAI-compatible path may still require harmless placeholders such as `OPENAI_API_KEY=dummy` and, for local embeddings, `LITELLM_DUMMY_API_KEY=dummy`. Those dummy values are only local client-compatibility placeholders; they do not secure the proxy. +- Keep real secrets out of `registered_models.*.api_key`; registered-model config can appear in application or worker logs and model metadata responses. +- Prefer environment or secret-manager handling for proxy credentials, keep `.env` out of version control, do not commit real proxy credentials in shared config files, and do not embed credentials or bearer tokens in `api_base`, `base_url`, or `baseUrl` URLs. + +This recipe also covers query and agent traffic that use `[completion].model`. If contextual chunking or parser vision traffic should use the same proxy, update `parser.contextual_chunking_model` and `[parser.vision].model` to proxy-backed registered model keys as well. + +These examples use HTTP for local Docker networking. Use HTTPS and authentication for remote or shared-network proxy endpoints. + ## Accessing Services - Morphik API: http://localhost:8000 diff --git a/morphik.docker.toml b/morphik.docker.toml index 9b90dc58..b9293b10 100644 --- a/morphik.docker.toml +++ b/morphik.docker.toml @@ -25,6 +25,37 @@ dev_user_id = "dev_user" # Default dev user ID openai_gpt4-1 = { model_name = "gpt-4.1" } openai_gpt4-1-mini = { model_name = "gpt-4.1-mini" } +# LiteLLM proxy / OpenAI-compatible endpoints +# - Put the proxy URL in api_base inside registered_models. +# - Reference the registered model key from [completion].model or [embedding].model. +# - Use the proxy service name, such as http://litellm:4000, when the proxy +# runs in the same Docker Compose project. +# - Use host.docker.internal only when the proxy runs on the Docker host. +# Linux Docker setups may require extra_hosts: ["host.docker.internal:host-gateway"] +# on every container that calls the proxy, at least morphik and worker. +# - Set OPENAI_API_KEY in the shared .env or on every proxy-calling service, +# including morphik and worker, for LiteLLM/OpenAI-compatible calls. +# - If an authenticated embedding proxy uses a local-looking URL such as +# localhost, 127.0.0.1, or host.docker.internal, Morphik forwards +# LITELLM_DUMMY_API_KEY for local embedding calls, so set +# LITELLM_DUMMY_API_KEY to the proxy key too. +# - If your local proxy has authentication disabled, LiteLLM may still require +# harmless placeholders such as OPENAI_API_KEY=dummy and +# LITELLM_DUMMY_API_KEY=dummy. +# Dummy values are only local client-compatibility placeholders; they do not +# secure the proxy. +# - Keep real secrets out of registered_models.*.api_key; registered-model +# config can appear in application or worker logs and model metadata responses. +# - Prefer environment or secret-manager handling for proxy credentials. +# - Do not commit real proxy credentials in shared config files. +# - Do not embed credentials or bearer tokens in api_base URLs. +# - Use HTTPS for remote or shared-network proxy endpoints. +# - [completion].model also covers query and agent traffic. If contextual +# chunking or parser vision traffic should use the proxy, update +# parser.contextual_chunking_model and [parser.vision].model separately. +# litellm_proxy_chat = { model_name = "openai/gpt-4o-mini", api_base = "http://litellm:4000" } +# litellm_proxy_embedding = { model_name = "openai/text-embedding-3-small", api_base = "http://litellm:4000" } + # Azure OpenAI models azure_gpt4 = { model_name = "gpt-4", api_base = "YOUR_AZURE_URL_HERE", api_version = "2023-05-15", deployment_id = "gpt-4-deployment" } azure_gpt35 = { model_name = "gpt-3.5-turbo", api_base = "YOUR_AZURE_URL_HERE", api_version = "2023-05-15", deployment_id = "gpt-35-turbo-deployment" } @@ -60,6 +91,7 @@ azure_embedding = { model_name = "text-embedding-ada-002", api_base = "YOUR_AZUR #### Component configurations #### [completion] +# LiteLLM proxy example: model = "litellm_proxy_chat" model = "openai_gpt4-1-mini" #"openai_gpt4-1-mini" # Reference to a key in registered_models default_max_tokens = "1000" default_temperature = 0.3 @@ -76,6 +108,9 @@ max_retries = 3 # Number of retries for database operations retry_delay = 1.0 # Initial delay between retries in seconds [embedding] +# LiteLLM proxy example: model = "litellm_proxy_embedding" +# Keep dimensions compatible with existing vectors, or plan re-ingestion, +# migration, a new collection/vector table, or a database reset. model = "openai_embedding" # Reference to registered model dimensions = 1536 similarity_metric = "cosine" diff --git a/morphik.toml b/morphik.toml index b328b974..9a17d4cc 100644 --- a/morphik.toml +++ b/morphik.toml @@ -19,6 +19,31 @@ dev_user_id = "dev_user" # Default dev user ID openai_gpt4-1 = { model_name = "gpt-4.1" } openai_gpt4-1-mini = { model_name = "gpt-4.1-mini" } +# LiteLLM proxy / OpenAI-compatible endpoints +# - Put the proxy URL in api_base inside registered_models. +# - Reference the registered model key from [completion].model or [embedding].model. +# - Set OPENAI_API_KEY in your environment to the proxy key for +# LiteLLM/OpenAI-compatible calls. +# - If an authenticated embedding proxy uses a local-looking URL such as +# localhost or 127.0.0.1, Morphik forwards LITELLM_DUMMY_API_KEY for local +# embedding calls, so set LITELLM_DUMMY_API_KEY to the proxy key too. +# - If your local proxy has authentication disabled, LiteLLM may still require +# harmless placeholders such as OPENAI_API_KEY=dummy and +# LITELLM_DUMMY_API_KEY=dummy. +# Dummy values are only local client-compatibility placeholders; they do not +# secure the proxy. +# - Keep real secrets out of registered_models.*.api_key; registered-model +# config can appear in application or worker logs and model metadata responses. +# - Prefer environment or secret-manager handling for proxy credentials. +# - Do not commit real proxy credentials in shared config files. +# - Do not embed credentials or bearer tokens in api_base URLs. +# - Use HTTPS for remote or shared-network proxy endpoints. +# - [completion].model also covers query and agent traffic. If contextual +# chunking or parser vision traffic should use the proxy, update +# parser.contextual_chunking_model and [parser.vision].model separately. +# litellm_proxy_chat = { model_name = "openai/gpt-4o-mini", api_base = "http://localhost:4000" } +# litellm_proxy_embedding = { model_name = "openai/text-embedding-3-small", api_base = "http://localhost:4000" } + # Azure OpenAI models azure_gpt4 = { model_name = "gpt-4", api_base = "YOUR_AZURE_URL_HERE", api_version = "2023-05-15", deployment_id = "gpt-4-deployment" } azure_gpt35 = { model_name = "gpt-3.5-turbo", api_base = "YOUR_AZURE_URL_HERE", api_version = "2023-05-15", deployment_id = "gpt-35-turbo-deployment" } @@ -59,6 +84,7 @@ azure_embedding = { model_name = "text-embedding-ada-002", api_base = "YOUR_AZUR #### Component configurations #### [completion] +# LiteLLM proxy example: model = "litellm_proxy_chat" model = "ollama_qwen_vision" #"openai_gpt4-1-mini" # Reference to a key in registered_models default_max_tokens = "1000" default_temperature = 0.3 @@ -75,6 +101,9 @@ max_retries = 3 # Number of retries for database operations retry_delay = 1.0 # Initial delay between retries in seconds [embedding] +# LiteLLM proxy example: model = "litellm_proxy_embedding" +# Keep dimensions compatible with existing vectors, or plan re-ingestion, +# migration, a new collection/vector table, or a database reset. model = "ollama_embedding" # Reference to registered model dimensions = 768 similarity_metric = "cosine"