diff --git a/.github/workflows/funnel-docs-check.yml b/.github/workflows/funnel-docs-check.yml new file mode 100644 index 00000000..4ddd38c9 --- /dev/null +++ b/.github/workflows/funnel-docs-check.yml @@ -0,0 +1,29 @@ +name: Check Funnel documentation sync + +on: + pull_request: + push: + branches: + - main + workflow_dispatch: + +permissions: + contents: read + +jobs: + check: + name: Test sync and build site + runs-on: ubuntu-latest + steps: + - name: Check out CALYPR docs + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Set up Python + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: '3.12' + - name: Test sync behavior + run: python -m unittest discover -s tests -v + - name: Install website dependencies + run: python -m pip install -r requirements.txt + - name: Build website + run: zensical build --clean diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 578827ce..ec20714e 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -23,9 +23,9 @@ jobs: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: - python-version: 3.x + python-version: '3.12' - run: pip install -r requirements.txt diff --git a/.github/workflows/sync-funnel-docs.yml b/.github/workflows/sync-funnel-docs.yml new file mode 100644 index 00000000..f214faea --- /dev/null +++ b/.github/workflows/sync-funnel-docs.yml @@ -0,0 +1,77 @@ +name: Update Funnel documentation + +on: + schedule: + - cron: '17 6 * * *' + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +concurrency: + group: sync-funnel-docs + cancel-in-progress: false + +jobs: + sync: + name: Prepare Funnel docs update + runs-on: ubuntu-latest + steps: + - name: Check out CALYPR docs + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Check out Funnel develop + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: calypr/funnel + ref: develop + path: .funnel-upstream + persist-credentials: false + + - name: Check Funnel revision + id: revision + run: | + revision=$(git -C .funnel-upstream rev-parse HEAD) + previous=$(python3 -c 'import json; print(json.load(open("docs/tools/funnel/.funnel-docs.json"))["revision"])') + echo "revision=$revision" >> "$GITHUB_OUTPUT" + if [ "$revision" != "$previous" ]; then + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + + - name: Set up Python + if: steps.revision.outputs.changed == 'true' + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: '3.12' + + - name: Test sync behavior + if: steps.revision.outputs.changed == 'true' + run: python -m unittest discover -s tests -v + + - name: Sync Funnel documentation + if: steps.revision.outputs.changed == 'true' + run: python scripts/sync_funnel_docs.py .funnel-upstream --revision "${{ steps.revision.outputs.revision }}" + + - name: Install website dependencies + if: steps.revision.outputs.changed == 'true' + run: python -m pip install -r requirements.txt + + - name: Build the website + if: steps.revision.outputs.changed == 'true' + run: zensical build --clean + + - name: Create or update pull request + if: steps.revision.outputs.changed == 'true' + uses: peter-evans/create-pull-request@22a9089034f40e5a961c8808d113e2c98fb63676 # v7 + with: + branch: automation/sync-funnel-docs + commit-message: 'docs: sync Funnel develop documentation' + title: 'docs: sync Funnel develop documentation' + body: | + Update the CALYPR site from [Funnel develop](https://github.com/calypr/funnel/tree/${{ steps.revision.outputs.revision }}) at `${{ steps.revision.outputs.revision }}`. + + Review the generated content and navigation before merging. This PR was prepared and built by the scheduled documentation sync. + delete-branch: true diff --git a/.gitignore b/.gitignore index f74ee97e..d9bc0c6e 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ venv site/ .idea/ attic/ + +/.funnel-upstream/ diff --git a/DEVELOPING.md b/DEVELOPING.md index 5f9a3614..901916d2 100644 --- a/DEVELOPING.md +++ b/DEVELOPING.md @@ -1,176 +1,57 @@ # Developing CALYPR Docs -This repo now uses a Zensical-only build pipeline. MkDocs, `.nav.yml`, and the old multirepo plugin are gone. +The site is built with Zensical from the content in `docs/`. Navigation is +configured in `zensical.toml`. -The main thing to understand is that the published site is built from a generated docs tree, not directly from `docs/`. +## Build locally -## Mental Model - -- `docs/` contains repo-authored source content. -- `.generated/docs/` is the staged build input that Zensical actually reads. -- `site/` is the rendered output from `zensical build`. -- `zensical.toml` is the only navigation source of truth. - -The build pipeline is: - -1. Copy repo-authored docs from `docs/` into `.generated/docs/` -2. Import selected upstream docs from `git-drs` and `syfon` -3. Apply a few link rewrites needed for imported pages -4. Add generated assets like `termynal.css` and `termynal.js` -5. Run Zensical against `.generated/docs` - -## Core Commands - -From the repo root: - -```bash +```sh python3 -m venv venv source venv/bin/activate pip install -r requirements.txt -make serve +make build ``` -Useful commands: - -- `make prepare`: rebuild `.generated/docs` -- `make build`: run a clean Zensical build -- `make serve`: prepare docs and start the local server -- `make update-branches`: update GitHub source links in authored docs to the preferred upstream branches - -If you want the raw commands: - -```bash -python scripts/prepare_docs.py -zensical build --clean -zensical serve -``` - -## Where Navigation Lives - -Navigation is fully defined in [zensical.toml](./zensical.toml). - -If you want to: - -- rename a sidebar label -- move a section -- add a new page to the nav -- change the Tools grouping - -edit the `nav = [...]` structure in `zensical.toml`. - -There are no `.nav.yml` files anymore. If you add a page under `docs/` and do not add it to `zensical.toml`, it will not appear in the intended nav structure. - -## Source of Truth Rules - -### Repo-authored pages - -These live in `docs/` and should be edited directly there. - -Examples: - -- `docs/index.md` -- `docs/calypr/...` -- `docs/tools/index.md` -- `docs/tools/funnel/...` -- `docs/tools/grip/...` - -### Imported tool pages - -These are not authored in this repo anymore. - -Current imported repos: - -- `git-drs` -- `syfon` - -Their imported file mappings live in [scripts/prepare_docs.py](./scripts/prepare_docs.py) under `REPO_IMPORTS`. +Use `make serve` for local development. The rendered site is written to `site/`. +The [GitHub Pages workflow](.github/workflows/publish.yml) builds and publishes +when changes reach `main`. -If you need to: +## Funnel documentation -- add a new imported page -- stop importing a page -- change where an imported page lands under `tools/...` +The canonical Funnel reference pages live in the `docs/` +directory on [Funnel's `develop` branch](https://github.com/calypr/funnel/tree/develop/docs). +A scheduled workflow checks that branch daily and opens or updates a pull request +with changed pages and generated navigation. Review and merge the pull request +to publish the update. The workflow also runs the sync tests and builds the site. -update `REPO_IMPORTS` and usually also update `zensical.toml`. +The Funnel landing and download pages remain CALYPR site content. Reference +pages, section overviews, navigation, and documentation assets come from Funnel. +The public installer is copied from Funnel's `install.sh`. -Do not hand-edit imported pages under `.generated/docs`. They are build artifacts and will be replaced on the next `make prepare`. +The sync reads page titles from Markdown headings and keeps the existing +Overview URL by importing `docs/README.md` as `docs/docs.md`. Repository-source +links are rewritten to the imported commit on GitHub. Retired pages are removed, +including Elasticsearch. Private `docs/internal/` content and hidden files are +excluded. Unknown sections and Hugo shortcodes stop the sync before publication. -## How Upstream Imports Work +The earlier exceptions for the GCP Batch, Kubernetes, and S3 guides have been +removed: their reconciled content is now in Funnel `develop`. -The importer prefers local sibling repos by default: +To run the sync locally against a Funnel checkout: -- `../git-drs` -- `../syfon` - -So on a typical local machine, the imported content comes from whatever is currently checked out in those repos. - -If local sibling repos are unavailable, the importer falls back to GitHub and picks the first available branch from [scripts/branch_config.json](./scripts/branch_config.json). - -Current branch preference order: - -- `development` -- `develop` -- `main` -- `master` - -The last import decision is written to `.generated/imports.json`. - -Environment overrides: - -- `CALYPR_IMPORT_PREFER_LOCAL=0`: force remote import mode -- `CALYPR_IMPORT_GIT_DRS_DIR=/path/to/git-drs`: override local `git-drs` source -- `CALYPR_IMPORT_SYFON_DIR=/path/to/syfon`: override local `syfon` source - -## When You Add or Change Pages - -### Adding a repo-authored page - -1. Create the file under `docs/` -2. Add it to `zensical.toml` -3. Run `make serve` or `make build` - -### Adding an imported tool page - -1. Add the file mapping in `scripts/prepare_docs.py` -2. Add the page to `zensical.toml` -3. If the imported markdown links break after relocation, add a targeted rewrite in `MARKDOWN_REWRITES` -4. Run `make prepare` -5. Run `make build` - -## Styling and Overrides - -Site-level assets still come from the repo: - -- custom CSS: `docs/stylesheets/extra.css` -- custom JS: `docs/javascripts/nav-open-level1.js` -- theme overrides: `overrides/` - -Those are copied into `.generated/docs` during preparation and then used by Zensical. - -## Deployment Behavior - -Deployment uses the same pipeline as local builds: - -- GitHub Pages runs `python scripts/prepare_docs.py` and then `zensical build --clean` -- Netlify does the same - -That means deployment now depends on the import step succeeding. In CI, the importer will usually use remote GitHub repos rather than local sibling repos. - -## Files You Should Usually Ignore - -- `.generated/` -- `site/` -- `.cache/` - -These are build/runtime artifacts. - -## Known Caveat - -Zensical currently reports some warnings from imported and legacy docs content, especially around bracketed literals like: +```sh +python scripts/sync_funnel_docs.py /path/to/funnel --revision "$(git -C /path/to/funnel rev-parse HEAD)" +python -m unittest discover -s tests -v +make build +``` -- `[SIGNED]` -- `[0]` -- `[]string` -- `map[string]string` +Generated Funnel pages and the marked Funnel navigation block should be changed +in the Funnel repository, not edited manually here. The sync branch is reused +while an update PR is open. If the Funnel revision is unchanged, no PR is created. -Those are content/rendering warnings, not a sign that the Zensical migration is broken. +The workflow needs both the organization policy and the repository setting that +allow GitHub Actions to create pull requests, plus `contents: write` and +`pull-requests: write` job permissions. +GitHub places PR workflows created with `GITHUB_TOKEN` in an approval-required +state. The sync workflow runs its own tests and site build before creating the +PR; a repository writer can approve the PR workflows separately. diff --git a/README.md b/README.md index 9fc10980..d23c89d7 100644 --- a/README.md +++ b/README.md @@ -15,16 +15,14 @@ make serve The local build contract is: -- `make prepare` stages repo-authored docs plus imported upstream tool docs into `.generated/docs` - `make build` runs a clean Zensical build -- `make serve` stages the docs and starts the local Zensical server +- `make serve` starts the local Zensical server For repo maintenance details, navigation ownership, and import behavior, see [DEVELOPING.md](./DEVELOPING.md). If you prefer the raw commands: ```sh -python scripts/prepare_docs.py zensical serve ``` diff --git a/docs/tools/funnel/.funnel-docs.json b/docs/tools/funnel/.funnel-docs.json new file mode 100644 index 00000000..ddce6b34 --- /dev/null +++ b/docs/tools/funnel/.funnel-docs.json @@ -0,0 +1,60 @@ +{ + "source": "https://github.com/calypr/funnel", + "branch": "develop", + "source_directory": "docs", + "revision": "738ab75a7056321d82cf57d9781d0af5a4433da6", + "files": [ + "compute.md", + "compute/aws-batch.md", + "compute/deployment.md", + "compute/gcp-batch.md", + "compute/gpus.md", + "compute/grid-engine.md", + "compute/htcondor.md", + "compute/kubernetes-storage.md", + "compute/kubernetes-templates/README.md", + "compute/kubernetes-templates/role.yaml", + "compute/kubernetes-templates/rolebinding.yaml", + "compute/kubernetes-templates/worker-configmap.yaml", + "compute/kubernetes-templates/worker-pv.yaml", + "compute/kubernetes-templates/worker-pvc.yaml", + "compute/kubernetes.md", + "compute/pbs-torque.md", + "compute/slurm.md", + "databases.md", + "databases/badger.md", + "databases/boltdb.md", + "databases/datastore.md", + "databases/dynamodb.md", + "databases/mongodb.md", + "databases/postgres.md", + "development.md", + "development/ci.md", + "development/developers.md", + "development/diagnostics.md", + "development/go-packages.md", + "docs.md", + "events.md", + "events/file.md", + "events/kafka.md", + "integrations/nextflow.md", + "integrations/py-tes.md", + "integrations/remote-server.md", + "metrics.md", + "metrics/prometheus.md", + "security.md", + "security/advanced.md", + "security/basic.md", + "security/oauth2.md", + "storage.md", + "storage/ftp.md", + "storage/google-storage.md", + "storage/http.md", + "storage/local.md", + "storage/mounted-inputs.md", + "storage/s3.md", + "storage/swift.md", + "tasks.md", + "upgrading.md" + ] +} diff --git a/docs/tools/funnel/docs/_releases.md b/docs/tools/funnel/docs/_releases.md deleted file mode 100644 index 799e7336..00000000 --- a/docs/tools/funnel/docs/_releases.md +++ /dev/null @@ -1,7 +0,0 @@ -| Asset | Download | -| --- | --- | -| funnel-darwin-amd64-v0.11.12.tar.gz | [Download](https://github.com/calypr/funnel/releases/download/v0.11.12/funnel-darwin-amd64-v0.11.12.tar.gz) | -| funnel-darwin-arm64-v0.11.12.tar.gz | [Download](https://github.com/calypr/funnel/releases/download/v0.11.12/funnel-darwin-arm64-v0.11.12.tar.gz) | -| funnel-linux-amd64-v0.11.12.tar.gz | [Download](https://github.com/calypr/funnel/releases/download/v0.11.12/funnel-linux-amd64-v0.11.12.tar.gz) | -| funnel-linux-arm64-v0.11.12.tar.gz | [Download](https://github.com/calypr/funnel/releases/download/v0.11.12/funnel-linux-arm64-v0.11.12.tar.gz) | -| funnel-v0.11.12-checksums.txt | [Download](https://github.com/calypr/funnel/releases/download/v0.11.12/funnel-v0.11.12-checksums.txt) | diff --git a/docs/tools/funnel/docs/compute.md b/docs/tools/funnel/docs/compute.md new file mode 100644 index 00000000..a40b677a --- /dev/null +++ b/docs/tools/funnel/docs/compute.md @@ -0,0 +1,23 @@ +# Compute + +## Listing scheduler nodes + +The Funnel scheduler extension exposes `GET /v1/nodes`. The first request can +omit pagination parameters; it returns at most 500 nodes. Set `page_size` to a +value from 1 through 1000 to choose a page size. Each response includes +`next_page_token` when more nodes are available. Pass that opaque value as +`page_token` to fetch the next page. An empty `next_page_token` marks the end. + +```sh +curl --get 'http://localhost:8000/v1/nodes' \ + --data-urlencode 'page_size=100' + +# Use the next_page_token from the previous response. +curl --get 'http://localhost:8000/v1/nodes' \ + --data-urlencode 'page_size=100' \ + --data-urlencode 'page_token=' +``` + +The token is a live continuation cursor, not a snapshot. Concurrent node +changes can affect later pages. Clients that previously expected one response +to contain every node must follow `next_page_token` until it is empty. diff --git a/docs/tools/funnel/docs/compute/aws-batch.md b/docs/tools/funnel/docs/compute/aws-batch.md index bebc256d..7ed9c60e 100644 --- a/docs/tools/funnel/docs/compute/aws-batch.md +++ b/docs/tools/funnel/docs/compute/aws-batch.md @@ -1,11 +1,3 @@ ---- -title: AWS Batch -menu: - main: - parent: Compute - weight: 20 ---- - # AWS Batch This guide covers deploying a Funnel server that leverages [DynamoDB][0] for storage @@ -48,12 +40,36 @@ $ funnel aws batch create-all-resources --region us-west-2 This command will create a compute environment, job queue, IAM role and job definition. +### Pull private Amazon ECR images + +To run a task with a private ECR image, configure registry authentication for +the Docker client used by the Funnel worker. The worker also needs AWS +credentials with permission to pull the image. See the AWS guide to +[private registry authentication](https://docs.aws.amazon.com/AmazonECR/latest/userguide/registry_auth.html), +including the ECR Docker credential helper. + +### Local storage on AWS Batch + +Local file paths must be accessible to the Batch worker. A file on the Funnel +server is not automatically available to workers on other machines. Use a +shared filesystem mounted on the workers and configure its path in +`LocalStorage.AllowedDirs`. If local storage is not needed, disable the backend: + +```yaml +LocalStorage: + Disabled: true +``` + ## Configuring the Funnel Server -Below is an example configuration. Note that the `Key` -and `Secret` fields are left blank in the configuration of the components. This is because -Funnel will, by default, try to automatically load credentials from the environment. -Alternatively, you may explicitly set the credentials in the config. +For EC2-backed Batch jobs, Funnel records `instance_type` in the task log +metadata when the worker can reach the EC2 Instance Metadata Service using +IMDSv2. The field may be absent if metadata access is disabled or the container +cannot reach it. See the [AWS metadata access guidance](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html) +for container networking considerations. Jobs still run if metadata is +unavailable. This field helps estimate costs but is not a bill. + +Below is an example configuration. Funnel loads AWS credentials from the environment or its AWS credential provider chain when explicit credentials are omitted. ```YAML Database: "dynamodb" @@ -61,22 +77,20 @@ Compute: "aws-batch" EventWriters: - "log" -Dynamodb: +DynamoDB: TableBasename: "funnel" - Region: "us-west-2" - Key: "" - Secret: "" + AWSConfig: + Region: "us-west-2" -Batch: +AWSBatch: JobDefinition: "funnel-job-def" JobQueue: "funnel-job-queue" - Region: "us-west-2" - Key: "" - Secret: "" + AWSConfig: + Region: "us-west-2" AmazonS3: - Key: "" - Secret: "" + AWSConfig: + Region: "us-west-2" ``` ### Start the server @@ -85,6 +99,55 @@ AmazonS3: funnel server run --config /path/to/config.yaml ``` +## GPU jobs + +`Compute: aws-batch` selects Funnel's AWS Batch execution code. It is a backend identifier, not an EC2 instance type or GPU model. The Batch worker must also receive `--Compute aws-batch`; newly generated Funnel job definitions include this flag. Update existing job definitions before running GPU tasks. + +### Select the hardware + +Use the provider documentation to choose hardware and configure the queue before submitting a TES task: + +- [EC2 accelerated computing offerings](https://aws.amazon.com/ec2/instance-types/accelerated-computing/) lists instance families and their accelerator specifications. +- [AWS Batch GPU jobs](https://docs.aws.amazon.com/batch/latest/userguide/gpu-jobs.html) describes supported GPU instances and GPU reservations. EC2 accelerator offerings include hardware that Batch's GPU scheduling does not support, so check both pages. +- [GPU workload AMIs](https://docs.aws.amazon.com/batch/latest/userguide/batch-gpu-ami.html) describes the required worker-host image and driver setup. +- [Batch compute environment configuration](https://docs.aws.amazon.com/batch/latest/userguide/create-compute-environment-managed-ec2.html) describes allowed instance types and capacity settings. + +To require a particular GPU model, use a queue whose associated compute environments permit only matching hardware. Naming a queue `a100-queue` does not enforce that constraint. The administrator must configure its compute environments accordingly and check availability and quotas in the selected AWS Region. + +### Request GPUs through TES + +For example, add these resource settings to a TES task: + +```json +{ + "resources": { + "backend_parameters_strict": true, + "backend_parameters": { + "gpu_count": "2", + "job_queue": "a100-queue", + "job_definition": "funnel-gpu-worker:2" + } + } +} +``` + +The queue and job definition in this example are deployment-defined names, not built-in Funnel presets. The server's AWS identity needs permission to submit the selected job. + +| Funnel setting | AWS behavior | +| --- | --- | +| `gpu_count` | Sets the `GPU` entry in the submitted job's `containerOverrides.resourceRequirements`. | +| `job_queue` | Selects the Batch queue; defaults to `AWSBatch.JobQueue`. | +| `job_definition` | Selects the worker job definition; defaults to `AWSBatch.JobDefinition`. | +| `AWSBatch.AWSConfig.Region` | Selects the AWS Region for Batch API requests. | + +### Pass allocated GPUs to the task container + +AWS Batch reserves devices for the Funnel worker container. That worker launches the task container through the host Docker socket. Funnel reads the worker's allocated device IDs from `NVIDIA_VISIBLE_DEVICES`, validates them, and passes those exact IDs to Docker's `--gpus` option. The TES user supplies the request, not the allocated device IDs. AWS assigns the IDs at runtime. + +A positive `gpu_count` with a missing, malformed, or mismatched allocation fails execution. Funnel does not substitute arbitrary host GPUs. Custom `Worker.Container.RunCommand` templates must include `{{.GPUArgs}}` to forward the allocation. + +See [GPU tasks](gpus.md) for a complete submission example, request validation, and deployment verification. Physical GPU access must be verified on the deployment; request-mapping tests do not exercise AWS GPU hardware. + ### Known issues The `Task.Resources.DiskGb` field does not have any effect. See [issue 317](https://github.com/ohsu-comp-bio/funnel/issues/317). diff --git a/docs/tools/funnel/docs/compute/deployment.md b/docs/tools/funnel/docs/compute/deployment.md index 2ea266f0..d1052a81 100644 --- a/docs/tools/funnel/docs/compute/deployment.md +++ b/docs/tools/funnel/docs/compute/deployment.md @@ -1,15 +1,7 @@ ---- -title: Deploying a cluster -menu: - main: - parent: Compute - weight: -50 ---- - # Deploying a cluster -This guide describes the basics of starting a cluster of Funnel nodes. -This guide is a work in progress. +This guide describes the basics of starting a cluster of Funnel nodes. +Funnel provides the application, container images, and configuration documentation. Operators maintain their own provisioning scripts, deployment manifests, service definitions, upgrades, and infrastructure operations. The former bundled GCE provisioning scripts and systemd examples have been removed. A node is a service which runs on each machine in a cluster. The node connects to the Funnel server and reports @@ -28,8 +20,24 @@ Nodes are available via the `funnel node` command. To start a node, run funnel node run --config node.config.yml ``` +When server authentication is enabled, remote nodes must use administrator +credentials in `RPCClient.Credential` to register and update their scheduler +records. Set `Admin: true` for the corresponding server Basic authentication +credential. Ordinary task-submitter credentials cannot create, update, or delete +scheduler nodes. + To activate the Funnel scheduler, use the `manual` backend in the config. +When upgrading to the task-claim scheduler, stop all older Funnel server +instances before starting the new version. Older schedulers do not consult +claims and can assign the same task to another node. Check the server logs for +conflicting existing assignments before restarting task submission. + +With BoltDB, PostgreSQL, and MongoDB, the scheduler records a task claim and its +node reservation together. MongoDB requires a replica set or sharded cluster +for these transactions. Standalone MongoDB deployments must be converted before +upgrading; Funnel rejects them at startup. See the [MongoDB configuration](../databases/mongodb.md). + The available scheduler and node config: ```yaml # Activate the Funnel scheduler. @@ -57,7 +65,7 @@ Node: # -1 means there is no timeout. 0 means timeout immediately after the first task. Timeout: -1s - # A Node will automatically try to detect what resources are available to it. + # A Node will automatically try to detect what resources are available to it. # Defining Resources in the Node configuration overrides this behavior. Resources: # CPUs available. @@ -77,3 +85,190 @@ Logger: # Write logs to this path. If empty, logs are written to stderr. OutputFile: "" ``` + +### Multiple instances sharing a database + +With the `manual` compute backend, multiple Funnel server and scheduler instances +can share one PostgreSQL database or MongoDB database. MongoDB requires a replica +set or sharded cluster. BoltDB is an embedded database and is not suitable for +sharing between hosts. + +The shared database provides one logical task queue for the connected nodes. +Schedulers can select the same queued task, but a transaction commits its +ownership and node capacity reservation together. Competing assignments cannot +claim the same task or reserve more capacity than the node has. Workers also +acquire execution permission before preparing files or starting an executor. + +Nodes can run at different sites if they can reach the Funnel services and the +storage their tasks require. A shared database does not automatically restrict +tasks to a particular site. Configure placement constraints and access to task +data and credentials for the intended worker pool. All participating servers, +nodes, and workers must use the upgraded version described below. + +Adding scheduler instances does not provide unlimited throughput. Instances can +read overlapping queued work, and assignments to the same node still coordinate +through the database. Ownership and execution admission do not guarantee +exactly-once process execution or external side effects after a crash. See +[execution admission](#execution-admission) for recovery limits. + +### Assignment storage upgrade + +BoltDB, PostgreSQL, and MongoDB use persisted task claims as the authoritative assignment +record. At first startup after this upgrade, Funnel imports active assignments +from existing node records in a transaction. Conflicting owners or a claim with +no reservation on an existing owner prevent startup and must be investigated. +Claims whose owner node has disappeared are retained for scheduler recovery. +The migration records completion and does not rescan all nodes during normal +assignment. + +Stop all older Funnel server and scheduler instances before upgrading. Node +heartbeat requests can update node status, but cannot create assignments or +remove active claims by changing `task_ids`. Funnel derives that list from the +claims. Terminal task states remove entries from the active node index while +preserving the durable ownership record. + +### Heartbeats, scheduling snapshots, and reservations + +For all three scheduler backends, `last_ping` is stored separately from resource +reservations. A routine heartbeat refreshes that timestamp without changing the +node's configuration/reservation version. Assignment and completion do not +refresh liveness. Resource, metadata, drain, and other configuration changes +still check that version. Scheduler health transitions also check the observed +heartbeat before marking a node dead or gone. + +Each scheduling pass reads one list of candidate nodes containing scheduling +fields, without expanding every node's task assignments. Successful assignments +and failed reservation attempts refresh the affected candidate. The database +validates current eligibility and capacity before committing an assignment. +Candidates can become stale during a pass; database checks remain authoritative. + +The default placement algorithm scans eligible candidates once and chooses the +lowest average CPU and RAM score. Each resource score is +`(available + requested) / total`. Equal scores retain candidate order. +Queue reads fetch task IDs, creation times, states, and requested resources +rather than full task payloads. PostgreSQL uses a partial queue-order index for +queued tasks. Multiple schedulers can still observe the same queued work; +atomic ownership checks arbitrate their reservation attempts. + +An allocation records the resource quantities reserved at assignment. Terminal +state writes release that allocation and update the node's resource usage in +the same database transaction. Repeated terminal writes do not release it twice. +Available capacity is derived from total capacity minus active usage, including +when a node's reported capacity shrinks below its existing reservations. + +A completed allocation cannot be reactivated by changing its task back to a +nonterminal state. Submit a new task to retry it. Resource accounting follows +TES terminal states, including cancellation; this does not establish that an +unreachable runtime has physically stopped its process. The design does not add +automatic reassignment or exactly-once process execution. + +Active task IDs are reconstructed from indexed claims for worker and public +node reads. Reservations and releases update a small resource budget and its +version without rewriting the descriptive node record or a growing task-ID +array. PostgreSQL keeps these values in dedicated columns, MongoDB updates the +budget fields, and BoltDB keeps a separate budget record. The public node API +continues to return task IDs and current available resources. + +Startup migrations snapshot existing active allocations, reconstruct resource +totals, and clear legacy task-ID arrays after importing their assignments. +PostgreSQL uses transactions scoped to the task and candidate node during +normal reservation/release. MongoDB uses snapshot transactions +with majority writes and retries write conflicts against fresh task and node +records. BoltDB retains its embedded single-writer transaction model. None of +these backends requires an external event log. + +### Execution admission + +Workers using BoltDB, PostgreSQL, or MongoDB acquire a durable execution claim +before preparing files or starting any executor. The database checks the task +state and records the claim with `INITIALIZING` in one transaction. After task +preparation, the worker sets `RUNNING` just before it dispatches the first executor. +This transition reports dispatch, not confirmation that a container has started. +Only the worker that receives a successful acquisition proceeds. A second worker +observes the task until it reaches a terminal state, without preparing files, +running an executor, or writing task events. This prevents a duplicate batch worker from +signaling successful completion while the original execution is still active. +An observer exiting successfully means observation finished; the TES task state +still reports whether execution completed, failed, or was canceled. +A failed claim request or an unsupported claim endpoint cannot authorize a launch. +Local tasks passed directly from a file or the command line do not use this +distributed claim. + +Claims have no expiration and do not transfer automatically. If a worker crashes +after acquiring its claim, or loses the successful response, another worker +cannot rerun that task ID. Kubernetes workers record their pod identity atomically +with the claim. Funnel submits worker pods with `restartPolicy: Never`, including +when a custom worker template specifies `OnFailure`. The Job's `backoffLimit` is +preserved, so failures before a claim is acquired can still start a replacement pod. +For worker pods with sidecars, Funnel tracks the selected worker container. If +that container fails before acquiring a claim, Funnel retires the pod so the +Job can retry even when a sidecar remains running. + +If the worker container that owns a claim exits, or its pod is lost before the task reaches a terminal state, +the Kubernetes reconciler records `SYSTEM_ERROR` and performs the configured +resource cleanup. It checks the pod UID and controlling Job UID before treating +pod termination as evidence that the claimed worker exited. A missing owner pod +or a different UID at its recorded name means that the owner was lost. It does not replay +the task or let a replacement worker take over its claim. A duplicate worker +continues observing the task while the original worker is alive. + +Claims created before this upgrade, manually launched workers without the pod +identity environment, and other compute backends do not have this pod ownership +information. Those tasks and duplicate observers can remain active until a user +cancels the task or existing backend failure recovery records a failure. An +unavailable Kubernetes API defers owner reconciliation. Failing a task after its +pod is deleted does not prove that a process on an unreachable node has stopped. +Inspect the original runtime +before submitting replacement work with a new task ID. Changing the old task's +state or retrying its claim does not grant another execution. + +This protects execution admission. It does not recover running containers, +stop an unreachable process, or guarantee exactly-once writes to external +systems. Resource release still follows TES terminal state and does not prove +that an unreachable runtime has stopped. + +Stop older servers, workers, and node processes before upgrading. Older workers +do not acquire execution claims and can bypass this protection. Initialize the +database with the upgraded server, then start upgraded workers. An upgraded RPC +worker requires an upgraded server; mixed-version rollouts do not provide this +protection. Existing running +and terminal tasks are treated as having consumed execution admission during +the startup migration. + +The claim is an internal worker operation. It does not change the TES task +schema or introduce a public TES endpoint. Worker RPC authentication and task +access rules also apply to acquisition. Database-backed workers using an +unsupported claim implementation fail before execution rather than silently +running without this guard. Datastore, DynamoDB, and Badger do not currently +implement execution admission; use BoltDB, PostgreSQL, or MongoDB for remote +workers with this version. Custom callers of `DefaultWorker` must supply an +`ExecutionClaimer` for non-local task readers. + +## Container command templates + +Task-provided image, name, workdir, volume paths, and environment or label entries are bound as literal runtime arguments. Quoted and unquoted scalar placeholders both preserve spaces and quotes in task values. Runtime flags belong in the administrator's template, rather than in task fields. Image identifiers cannot start with `-` or contain whitespace or control characters. Docker volume paths cannot contain `:` or NUL. + +Use `{{.GetEnvArgs}}` and `{{.GetTagArgs}}` to expand environment variables and opted-in task labels. Direct `range` loops over `Env` and `Tags` are also supported. These maps use protected string keys; custom templates using `index .Env "KEY"` or `index .Tags "KEY"` must migrate to the helpers or a `range` with `eq`. Avoid applying string truncation or serialization to runtime argument placeholders. Task command arrays continue to be forwarded separately. + +### Trusted HPC submission records + +Slurm, PBS, HTCondor, and Grid Engine require `HPCStateDir`, an absolute path to +private, persistent controller storage (mode `0700`). Keep it outside +`Worker.WorkDir`, `Worker.ScratchPath`, and every enabled +`LocalStorage.AllowedDirs` root. Use a bounded local-storage allowlist, or disable +local storage. For example, use `/var/lib/funnel/controller` for `HPCStateDir` +and `/var/lib/funnel/tasks` for task data. + +The controller records the scheduler job ID returned by submission there. +Cancellation and reconciliation use that record; provider IDs in task logs are +for display. Controllers managing the same jobs must share this directory, and +it must survive restarts. Do not mount it into executor containers. + +Jobs submitted before this upgrade have no trusted record. Cancel those jobs +with the scheduler's own command before upgrading; Funnel will refuse to use +legacy task metadata as cancellation authority. Preserve the records while jobs +can still be managed. Back up this directory with the task database. + +Output publication copies confined task files into a private temporary snapshot +before uploading. Allow local disk space and I/O for an additional copy of the +outputs during publication. diff --git a/docs/tools/funnel/docs/compute/gcp-batch.md b/docs/tools/funnel/docs/compute/gcp-batch.md index cb18f772..eedf7b21 100644 --- a/docs/tools/funnel/docs/compute/gcp-batch.md +++ b/docs/tools/funnel/docs/compute/gcp-batch.md @@ -1,11 +1,3 @@ ---- -title: GCP Batch -menu: - main: - parent: Compute - weight: 20 ---- - # Overview The following steps illustrate how to run TES tasks via GCP Batch utilizing Google Storage Buckets. @@ -122,10 +114,63 @@ gsutil cat gs://my-bucket/output/README.md.sha256 9b9916cea5348edd6ad78893231edb81fc96772d1dd99fae9c2a64f84646cb1c /input/README.md ``` +# GPU jobs + +`Compute: gcp-batch` selects Funnel's GCP Batch backend. Hardware selection uses the task's native GCP accelerator and machine-type settings. `GCPBatch.Location` selects the Batch job region; it does not select a GPU model. + +## Select the hardware and location + +Choose compatible accelerator, machine, and location values using Google's documentation: + +- [GPU machine types](https://docs.cloud.google.com/compute/docs/gpus) lists GPU offerings and their associated machine types. +- [GPU locations](https://docs.cloud.google.com/compute/docs/regions-zones/gpu-regions-zones) lists where each offering is available. Select a compatible region in `GCPBatch.Location`. +- [Batch GPU jobs](https://docs.cloud.google.com/batch/docs/create-run-job-gpus) describes supported configurations, GPU drivers, images, and container prerequisites. +- [Compute Engine allocation quotas](https://docs.cloud.google.com/compute/resource-usage) describes project quota requirements. Availability in a region does not imply available capacity or sufficient quota. + +Funnel passes the requested names to Google Cloud. It does not translate GPU marketing names or maintain a catalogue of compatible combinations. The provider validates the requested combination. + +## Request GPUs through TES + +For example, add these resource settings to a TES task: + +```json +{ + "resources": { + "backend_parameters_strict": true, + "backend_parameters": { + "gpu_type": "nvidia-tesla-t4", + "gpu_count": "1", + "machine_type": "n1-standard-4", + "install_gpu_drivers": "true" + } + } +} +``` + +The values illustrate the format. Check their compatibility and availability for your project and selected location before submitting. + +| Funnel setting | GCP Batch behavior | +| --- | --- | +| `gpu_type` | Sets `allocationPolicy.instances[].policy.accelerators[].type`. | +| `gpu_count` | Sets the accelerator `count`; requires a positive integer. | +| `machine_type` | Sets the instance policy's `machineType`; optional. | +| `install_gpu_drivers` | Sets `allocationPolicy.instances[].installGpuDrivers`; defaults to `true` for GPU requests. | +| `GCPBatch.Location` | Selects the region in the Batch job's parent resource. | + +`gpu_type` and `gpu_count` are required together. The optional `machine_type` and `install_gpu_drivers` parameters require that GPU pair. Setting `install_gpu_drivers` to `false` means the selected VM image must already provide the drivers and container tooling required by Google Batch. + +## Make GPUs available to the executor + +GCP Batch runs Funnel's TES executors directly as container runnables. Funnel adds `--gpus all` to those runnables' container options so they can access the GPUs attached to the job VM. This path does not use the nested Funnel Docker worker or AWS's `NVIDIA_VISIBLE_DEVICES` forwarding mechanism. + +See [GPU tasks](gpus.md) for a complete TES submission example and deployment verification. Request-mapping tests inspect the Batch API payload; physical GPU access still requires a live job on the deployment. + # Nextflow Nextflow tasks can specify a working directory via the TES executor `workdir` field, which maps to Docker's `--workdir` flag inside the GCP Batch container: +GCP Batch workdirs must be absolute paths containing only ASCII letters, digits, `/`, `.`, `_`, and `-`. Funnel rejects whitespace, quotes, and other characters before submitting the job because Batch exposes container options as a string rather than an argument array. Other Funnel backends can support paths with spaces. + ```json { "executors": [ diff --git a/docs/tools/funnel/docs/compute/gpus.md b/docs/tools/funnel/docs/compute/gpus.md new file mode 100644 index 00000000..8e35a8f3 --- /dev/null +++ b/docs/tools/funnel/docs/compute/gpus.md @@ -0,0 +1,173 @@ +# GPU tasks + +Funnel accepts GPU settings through TES `resources.backend_parameters`. The supported keys depend on the compute backend. Funnel passes hardware names to that backend; it does not maintain a GPU model catalogue. + +Set `backend_parameters_strict` to `true` when the task requires these settings. Unsupported strict parameters reject task creation. Unsupported optional parameters produce a server warning and are removed from the stored task. Parameter names are case-insensitive. Supplying the same name with different casing is an error. + +Use `GET /v1/service-info` to read `tesResources_backend_parameters`, the list of supported keys for this deployment. + +## Submit a GPU task + +Each backend example below supplies the `resources` portion of a TES task. Add it to a task with a compatible container image and command. For example, save this local Docker task as `gpu-task.json`: + +```json +{ + "name": "GPU visibility check", + "resources": { + "backend_parameters_strict": true, + "backend_parameters": {"gpu_count": "1"} + }, + "executors": [ + {"image": "ubuntu:22.04", "command": ["nvidia-smi", "-L"]} + ] +} +``` + +The NVIDIA runtime must provide `nvidia-smi` and driver access to this container. For another backend, replace `resources` with that backend's example and use the GPU image required by the deployment. + +Submit the file to Funnel: + +```sh +curl --fail-with-body -X POST http://localhost:8000/v1/tasks \ + -H 'Content-Type: application/json' \ + --data-binary @gpu-task.json +``` + +Use the returned task ID to inspect its status and executor logs: + +```sh +curl --fail-with-body 'http://localhost:8000/v1/tasks/TASK_ID?view=FULL' +``` + +Replace the endpoint and authentication options with those for your deployment. A successful task reports the visible GPU identities in its executor stdout. + +## Kubernetes + +```json +{ + "resources": { + "backend_parameters_strict": true, + "backend_parameters": { + "gpu_count": "1", + "gpu_resource": "nvidia.com/gpu", + "gpu_node_selector": "accelerator=a100" + } + } +} +``` + +`gpu_count` is a positive integer. `gpu_resource` defaults to `nvidia.com/gpu`. `gpu_node_selector` accepts comma-separated `key=value` node labels, such as `accelerator=a100,vendor=nvidia`. The cluster administrator defines these labels. These labels select the required hardware. A conflicting administrator or executor-template selector is rejected. + +Funnel sets the GPU request and limit on the executor container, including when an administrator supplies a custom executor Job template. The worker and sidecar containers do not receive the task's GPU request. A conflicting GPU request or limit already present in the executor template is rejected. + +The cluster requires the appropriate GPU device plugin, drivers, and container runtime configuration. Kubernetes performs placement and device allocation. A resource request can remain pending when matching capacity is unavailable. + +## AWS Batch + +See the [AWS Batch GPU settings](aws-batch.md#gpu-jobs) for hardware offerings, queue configuration, and worker setup. + +```json +{ + "resources": { + "backend_parameters_strict": true, + "backend_parameters": { + "gpu_count": "2", + "job_queue": "a100-queue", + "job_definition": "funnel-gpu-worker:2" + } + } +} +``` + +`gpu_count` adds a Batch GPU resource requirement. Optional `job_queue` and `job_definition` override the configured defaults. Each accepts an AWS resource name or ARN. The server's AWS identity must have permission to submit the selected job. Administrators control which hardware is available through the queue's compute environments and allowed instance types. + +Batch reserves GPUs for the Funnel worker container. Funnel launches task containers through the host Docker socket, so it must also forward the devices allocated to the worker. Funnel reads Batch's `NVIDIA_VISIBLE_DEVICES` allocation and requests those exact devices for the executor. A missing or incompatible allocation fails execution instead of selecting arbitrary host GPUs. + +The Batch worker command must include `--Compute aws-batch`. Newly generated Funnel job definitions include it. Update existing job definitions before submitting GPU tasks. + +Use a GPU-capable compute environment, worker image, Docker runtime, and task image. GPU scheduling in AWS Batch supports NVIDIA hardware. Custom Docker run templates must include `{{.GPUArgs}}`. + +## GCP Batch + +See the [GCP Batch GPU settings](gcp-batch.md#gpu-jobs) for accelerator offerings, machine types, locations, and quotas. + +```json +{ + "resources": { + "backend_parameters_strict": true, + "backend_parameters": { + "gpu_type": "nvidia-tesla-t4", + "gpu_count": "1", + "machine_type": "n1-standard-4", + "install_gpu_drivers": "true" + } + } +} +``` + +`gpu_type` and `gpu_count` are required together. The type is a native GCP accelerator name and the count is a positive integer. `machine_type` optionally selects a compatible VM type. `install_gpu_drivers` defaults to `true`; set it to `false` only when the VM image already provides the required drivers and container toolkit. Both optional fields require a GPU request. + +Funnel sets the Batch allocation policy and enables GPU access on each executor runnable. The selected location, machine type, accelerator, quota, and image must be compatible. GCP validates their compatibility. + +## Slurm + +The administrator exposes native Slurm settings through a validated allowlist and a submission template. For example: + +```yaml +Slurm: + BackendParameters: + gres: '[A-Za-z0-9_:-]+' + partition: '[A-Za-z0-9_-]+' + TemplateFile: /etc/funnel/slurm-template.txt +``` + +Each regular expression must match the entire parameter value. Values containing control characters are rejected. Use narrow expressions because parameter values become part of the scheduler's submission file. Only Slurm loads this allowlist; the shared configuration type's `BackendParameters` field is not supported by PBS or HTCondor. + +Add these directives to the submission template: + +```text +{{with index .BackendParameters "gres"}}#SBATCH --gres={{.}}{{end}} +{{with index .BackendParameters "partition"}}#SBATCH --partition={{.}}{{end}} +``` + +TES resource settings for a deployment with an `a100` GRES type and a `gpu` partition: + +```json +{ + "resources": { + "backend_parameters_strict": true, + "backend_parameters": {"gres": "gpu:a100:2", "partition": "gpu"} + } +} +``` + +Templates receive lowercase parameter names. The Slurm administrator defines the supported GRES types and partitions. The worker must load the Slurm configuration, for example through `funnel worker run --Compute slurm --taskID ...`. The default Slurm template includes this flag. + +Before starting the worker, the template or a site allocation hook must export `FUNNEL_GPU_DEVICES`. Its value is a comma-separated list of allocated physical GPU IDs or UUIDs as seen by the host Docker daemon. Prefer UUIDs. Slurm's `CUDA_VISIBLE_DEVICES` indices can be remapped inside cgroups; do not copy them blindly as host Docker device IDs. Funnel forwards `FUNNEL_GPU_DEVICES` to Docker. + +A submission directive alone reserves resources but does not provide container access. Verify the allocation hook on the cluster. Custom Docker run templates must include `{{.GPUArgs}}`. + +## Local Docker + +Local execution accepts `gpu_count` or `gpu_devices`. `gpu_devices` is a comma-separated list of physical GPU indices or NVIDIA GPU UUIDs. If both fields are present, the count must match the list length. Local Docker requires the NVIDIA Container Toolkit and compatible drivers. Local execution does not reserve GPUs between concurrent tasks. + +Local TES resource settings for two specific devices: + +```json +{ + "resources": { + "backend_parameters_strict": true, + "backend_parameters": {"gpu_devices": "0,2"} + } +} +``` + +To let Docker select two devices instead, use `"backend_parameters": {"gpu_count": "2"}`. Start Funnel with `--Compute local` for these examples. + +## Unsupported backends + +GPU backend parameters are not supported by PBS, Grid Engine, HTCondor, or Funnel's manual scheduler. These backends advertise no GPU keys. Set `backend_parameters_strict` to `true` to reject a task that requires unsupported settings. Optional unsupported settings produce a warning and are removed according to TES semantics. + +## Deployment verification + +Request propagation tests do not prove that a physical GPU is available. Before using a deployment for GPU workflows, submit a task with its required GPU parameters and a compatible image that runs `nvidia-smi -L`. Confirm that the model and number of visible devices match the request. Also run concurrent tasks to verify that the scheduler allocates distinct devices where exclusive allocation is expected. diff --git a/docs/tools/funnel/docs/compute/grid-engine.md b/docs/tools/funnel/docs/compute/grid-engine.md index f885c55b..081ad822 100644 --- a/docs/tools/funnel/docs/compute/grid-engine.md +++ b/docs/tools/funnel/docs/compute/grid-engine.md @@ -1,46 +1,23 @@ ---- -title: Grid Engine ---- # Grid Engine -Funnel can be configured to submit workers to [Grid Engine](https://gridscheduler.sourceforge.net/) by making calls +Funnel can be configured to submit workers to [Grid Engine][ge] by making calls to `qsub`. The Funnel server needs to run on a submission node. Configure Funnel to use Grid Engine by including the following config: It is recommended to update the submit file template so that the -`funnel worker run` command takes a config file as an argument: +`funnel worker run` command takes a config file as an argument +(e.g. `funnel worker run --config /opt/funnel_config.yml --taskID {{.TaskId}}`) -``` -{% raw %} -funnel worker run --config /opt/funnel_config.yml --taskID {{.TaskId}} -{% endraw %} -``` +Download the [Grid Engine submit template](https://github.com/calypr/funnel/blob/738ab75a7056321d82cf57d9781d0af5a4433da6/website/static/funnel-config-examples/gridengine-template.txt) +to `/etc/funnel/gridengine-template.txt`, then configure Funnel to read it: -```YAML -{% raw %} +```yaml Compute: gridengine - GridEngine: - Template: | - #!/bin/bash - #$ -N {{.TaskId}} - #$ -o {{.WorkDir}}/funnel-stdout - #$ -e {{.WorkDir}}/funnel-stderr - {{if ne .Cpus 0 -}} - {{printf "#$ -pe mpi %d" .Cpus}} - {{- end}} - {{if ne .RamGb 0.0 -}} - {{printf "#$ -l h_vmem=%.0fG" .RamGb}} - {{- end}} - {{if ne .DiskGb 0.0 -}} - {{printf "#$ -l h_fsize=%.0fG" .DiskGb}} - {{- end}} - funnel worker run --taskID {{.TaskId}} -{% endraw %} + TemplateFile: /etc/funnel/gridengine-template.txt ``` - The following variables are available for use in the template: | Variable | Description | @@ -54,3 +31,4 @@ The following variables are available for use in the template: See https://golang.org/pkg/text/template for information on creating templates. +[ge]: http://gridscheduler.sourceforge.net/documentation.html diff --git a/docs/tools/funnel/docs/compute/htcondor.md b/docs/tools/funnel/docs/compute/htcondor.md index a6c8ebdd..c8b301e8 100644 --- a/docs/tools/funnel/docs/compute/htcondor.md +++ b/docs/tools/funnel/docs/compute/htcondor.md @@ -1,49 +1,22 @@ ---- -title: HTCondor -menu: - main: - parent: Compute - weight: 20 ---- # HTCondor -Funnel can be configured to submit workers to [HTCondor][htcondor] by making +Funnel can be configured to submit workers to [HTCondor][htcondor] by making calls to `condor_submit`. The Funnel server needs to run on a submission node. Configure Funnel to use HTCondor by including the following config: It is recommended to update the submit file template so that the -`funnel worker run` command takes a config file as an argument {% raw %} -(e.g. `funnel worker run --config /opt/funnel_config.yml --taskID {{.TaskId}}`){% endraw %} +`funnel worker run` command takes a config file as an argument +(e.g. `funnel worker run --config /opt/funnel_config.yml --taskID {{.TaskId}}`) -```YAML -{% raw %} -Compute: htcondor +Download the [HTCondor submit template](https://github.com/calypr/funnel/blob/738ab75a7056321d82cf57d9781d0af5a4433da6/website/static/funnel-config-examples/htcondor-template.txt) +to `/etc/funnel/htcondor-template.txt`, then configure Funnel to read it: +```yaml +Compute: htcondor HTCondor: - Template: | - universe = vanilla - getenv = True - executable = funnel - arguments = worker run --taskID {{.TaskId}} - log = {{.WorkDir}}/condor-event-log - error = {{.WorkDir}}/funnel-stderr - output = {{.WorkDir}}/funnel-stdout - should_transfer_files = YES - when_to_transfer_output = ON_EXIT_OR_EVICT - {{if ne .Cpus 0 -}} - {{printf "request_cpus = %d" .Cpus}} - {{- end}} - {{if ne .RamGb 0.0 -}} - {{printf "request_memory = %.0f GB" .RamGb}} - {{- end}} - {{if ne .DiskGb 0.0 -}} - {{printf "request_disk = %.0f GB" .DiskGb}} - {{- end}} - - queue -{% endraw %} + TemplateFile: /etc/funnel/htcondor-template.txt ``` The following variables are available for use in the template: diff --git a/docs/tools/funnel/docs/compute/k8s-pvc.png b/docs/tools/funnel/docs/compute/k8s-pvc.png deleted file mode 100644 index 7a815d5b..00000000 Binary files a/docs/tools/funnel/docs/compute/k8s-pvc.png and /dev/null differ diff --git a/docs/tools/funnel/docs/compute/kubernetes-storage.md b/docs/tools/funnel/docs/compute/kubernetes-storage.md new file mode 100644 index 00000000..3b1801ee --- /dev/null +++ b/docs/tools/funnel/docs/compute/kubernetes-storage.md @@ -0,0 +1,213 @@ +# Kubernetes storage + +`Kubernetes.Storage` configures PVC-backed Kubernetes volumes for task workspaces and worker mounts. The `volumes` and `volumeMounts` fields use native Kubernetes objects, and configured volumes must use PVC sources. The `volumeClaimTemplates` and `persistentVolumeTemplates` fields contain native resource objects. `WorkspaceVolume` names the PVC-backed volume Funnel uses for the task workspace. `WorkerContainer` and `ExecutorContainer` select the containers that receive generated mounts when a Pod template has more than one regular container. + +When `Storage` is absent, Funnel keeps the existing `PVTemplate`, `PVCTemplate`, and Job template behavior. Explicit storage does not change TES inputs, outputs, or task paths. Configured GenericS3 mounted reads and writes require `Kubernetes.Storage` and a submitted storage plan. + +## Use an existing PVC + +Use `persistentVolumeClaim.claimName` for a claim that an operator already manages. Funnel does not create or delete that claim or its PV. + +```yaml +Worker: + WorkDir: /opt/funnel/funnel-work-dir +Kubernetes: + JobsNamespace: funnel-tasks + Storage: + WorkspaceVolume: workspace + volumes: + - name: workspace + persistentVolumeClaim: + claimName: shared-workspace +``` + +The claim must exist in `JobsNamespace` and support filesystem access from the worker and executor Pods. Funnel creates a task-specific workspace directory on that claim. `ReadWriteMany` supports placement across nodes. `ReadWriteOnce` requires compatible Pod placement. `ReadWriteOncePod` cannot serve the worker and executor Pods and is rejected. + +Use `volumes` entries for existing claims. Set `persistentVolumeClaim.claimName` to the claim's literal name. A PVC template creates a volume with the same logical name as the template. Do not add a matching `volumes` entry. A `volumeMounts` entry refers to either kind of volume through its `name` field. + +## Provision a PVC through a StorageClass + +Each PVC template creates a claim for one task and a volume with the same logical name. Set `WorkspaceVolume` to that name to use the claim for the task workspace. + +```yaml +Kubernetes: + JobsNamespace: funnel-tasks + Storage: + WorkspaceVolume: workspace + volumeClaimTemplates: + - metadata: + name: workspace + spec: + storageClassName: research-rwx + accessModes: + - ReadWriteMany + resources: + requests: + storage: "{{.DiskGb}}Gi" +``` + +For a task with ID `task-a`, this template creates the PVC `task-a-workspace` and a volume named `workspace`. Funnel sets the worker Job as the claim's owner. Kubernetes garbage collection removes the claim after the Job. The StorageClass determines provisioning and reclaim behavior. This configuration makes no PV API calls during normal submission or cleanup. + +Set each PVC template's `metadata.name` to a static logical volume name that is a DNS label, such as `workspace`. Funnel names the PVC `-`. Do not put `TaskId` in the PVC template's `metadata.name`. Omit `apiVersion` and `kind` to use `v1` and `PersistentVolumeClaim` defaults. Funnel infers an omitted `metadata.namespace` from the effective job namespace, which uses `JobsNamespace` or falls back to `Namespace`. Do not set `generateName`, owner references, or block volume mode. Each logical name must be unique and must not collide with a `volumes` entry. An existing-claim volume cannot refer to a generated claim name. + +Funnel renders Go templates in string values other than the PVC template's logical `metadata.name`, then decodes each object as a Kubernetes resource. It does not render an entire YAML document as template text. Quote templated quantity values so YAML keeps them as strings. Other fields can use `.TaskId`, `.Namespace`, and `.DiskGb`. They can also use `.Bucket`, `.Region`, and `.KmsKeyID` from the first GenericS3 entry when present. Those three values are empty when no GenericS3 entry exists. + +To migrate an existing task-specific PVC template, replace its `metadata.name` such as `work-{{.TaskId}}` with a static name such as `workspace`, then remove the `volumes` entry that points to the rendered claim name. If a PV template specifies `spec.claimRef.name`, remove that reference so Funnel fills the generated name, or change it to `{{.TaskId}}-workspace` for this example. Keep Go templates in other resource fields, including `spec.volumeName` and PV `metadata.name`. Existing version 4 storage plans remain compatible. The new naming rule applies to new task submissions. + +## Create an explicit PV and PVC pair + +Set `PVC.spec.volumeName` to the exact `metadata.name` of a `persistentVolumeTemplates` entry. The PVC template keeps its static logical name, while `spec.volumeName` can use `TaskId` to select a task-specific PV. Funnel fills a missing PV `spec.claimRef` with the generated PVC name and namespace. For task `task-a`, the example claim is `task-a-workspace`, and the PV claim reference uses that name. If the template provides a conflicting reference, Funnel rejects it before creating resources. + +```yaml +Kubernetes: + JobsNamespace: funnel-tasks + Storage: + WorkspaceVolume: workspace + volumeClaimTemplates: + - metadata: + name: workspace + spec: + storageClassName: "" + volumeName: disk-{{.TaskId}} + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi + persistentVolumeTemplates: + - apiVersion: v1 + kind: PersistentVolume + metadata: + name: disk-{{.TaskId}} + spec: + capacity: + storage: 20Gi + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: "" + hostPath: + path: /srv/funnel/{{.TaskId}} +``` + +The `hostPath` source makes this example suitable only for a local cluster whose worker and executor Pods use the same node and path. Use the deployment's native CSI or other PV source in a real cluster. Each PV template must be a cluster-scoped `v1` `PersistentVolume` with an explicit `metadata.name` that includes `TaskId`. Do not set `metadata.namespace`, `generateName`, owner references, or block volume mode. A PV template must have exactly one native PV source and link to exactly one PVC template. + +Funnel creates the PV and PVC with task ownership and preserves the existing cleanup lifecycle. Explicit PV storage adds a finalizer to the worker Job. Cleanup requests deletion of owned PVs and the Job before releasing that finalizer. A failed delete request leaves the plan available for retry. Reconciliation retries cleanup for canceled tasks even when the Job is still running or its deletion request failed. Submission checks persisted cancellation before and after resource creation and cleans up resources created during cancellation. The server needs permission to delete these PVs. Finish PV-backed tasks and clean up their resources before removing cluster-scoped deletion permissions. Kubernetes storage-protection finalizers and the storage driver's normal cleanup still apply. + +Before creating the worker Job, Funnel checks PV permissions with Kubernetes `SelfSubjectAccessReview`. Explicit PV templates require cluster-scoped `create` permission and `get` and `delete` permission for each generated PV name. The check uses the effective task configuration, including plugin overrides. A denied or unavailable authorization check rejects submission before creating the Job or its storage. Existing-claim and StorageClass-only configurations skip these PV checks. + +The Funnel server identity must be allowed to create `selfsubjectaccessreviews` in the `authorization.k8s.io` API group. Kubernetes normally grants this through its default authenticated-user roles. The submission check does not require PV `list` permission; the separate orphan cleanup sweep still needs it. Authorization can change after the check, so keep PV permissions until all PV-backed tasks have been cleaned up. + +To enable explicit PV storage, an administrator can apply the following RBAC resources. Replace the subject name and namespace with the service account used by the Funnel server Pod. These permissions supplement the server's existing namespaced permissions for Jobs, Pods, and PVCs. + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: funnel-pv-manager +rules: + - apiGroups: [""] + resources: [persistentvolumes] + verbs: [create, get, delete] + - apiGroups: [authorization.k8s.io] + resources: [selfsubjectaccessreviews] + verbs: [create] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: funnel-pv-manager +subjects: + - kind: ServiceAccount + name: funnel + namespace: funnel-tasks +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: funnel-pv-manager +``` + +## Add worker mounts + +`volumeMounts` contains native Kubernetes `volumeMount` objects. Use the configured volume's `name`. You can mount the same volume at multiple paths. + +```yaml +Kubernetes: + JobsNamespace: funnel-tasks + Storage: + WorkspaceVolume: workspace + volumes: + - name: workspace + persistentVolumeClaim: + claimName: shared-workspace + - name: reference-data + persistentVolumeClaim: + claimName: reference-datasets + volumeMounts: + - name: reference-data + mountPath: /mnt/reference + readOnly: true + subPath: genomes + - name: reference-data + mountPath: /mnt/annotations + readOnly: true + subPath: annotations +``` + +Funnel adds `volumeMounts` to the selected worker container. It generates the workspace mount for the worker and executor. Extra worker mounts do not expose data automatically at TES input paths. A storage backend must still prepare those inputs. Worker mount paths cannot overlap the task workspace, each other, or mounts already in the Job template. + +GenericS3 `MountedRead` and `MountedWrite` select a worker mount by volume name and, optionally, its static `subPath`. If one `volumeMounts` entry uses the selected volume, omit `SubPath`. If several entries use that volume, set `SubPath` to the exact `volumeMounts[].subPath` value. Set `SubPath: ""` to select a static root mount. A mount that uses `subPathExpr` can be selected only when it is the unique mount for that volume and `SubPath` is absent. + +The selected mount's authored `mountPath` is the S3 root. Funnel does not append the selected `SubPath` or the S3 `Prefix` to that path. A write mount must be writable. The underlying claim must also permit writes. + +## Configure Job templates + +With explicit storage, Funnel renders Job templates with `NeedsPVC: false` and adds resolved volumes and mounts afterward. Templates that already condition legacy storage on `NeedsPVC` keep their other settings. Remove template-defined volumes or mount paths that conflict with configured storage. + +Funnel selects the sole regular container by default. For templates with sidecars, set `Storage.WorkerContainer` and `Storage.ExecutorContainer` to the main container names. Funnel adds storage and worker configuration only to those containers. Init containers and sidecars keep their template-defined mounts. + +For a multi-container worker template without explicit storage, identify the worker with the pod-template annotation `funnel.calypr.org/worker-container`. Its value must match the worker container's name. Funnel records this selection when submitting the Job so worker failure handling does not wait for sidecars to exit. With explicit storage, `Storage.WorkerContainer` takes precedence over the annotation. + +The worker Job name must equal `TaskId`. `Worker.WorkDir` must be absolute. Per-task plugin overrides cannot change `JobsNamespace`, because a backend reconciles one configured Jobs namespace. + +`Kubernetes.WorkerImage` selects the image supplied to the worker Job template and skips server Pod discovery. When it is absent, Funnel keeps the existing image discovery behavior. Templates use `{{.Image}}` to consume that setting. + +## Ownership and upgrades + +Funnel labels resources it creates and records the worker Job UID. Creation retries accept only resources with matching ownership and specifications. Funnel rejects an existing resource with a conflicting owner or specification instead of adopting it. Operator-owned claims remain under operator control. + +Provisioning checks the worker Job before and after each resource creation. If cancellation removes the Job during creation, Funnel deletes a late-created owned PV. A process crash, an ambiguous create response, or a failed compensating deletion can leave an orphan. Recovery requires an orphan cleanup sweep with PV scanning enabled by a PV template. An existing-claim-only configuration does not scan cluster-scoped PVs. + +Funnel records a version 4 plan in the Job's `funnel.calypr.org/storage-plan` annotation and supplies the same plan through `FUNNEL_KUBERNETES_STORAGE_PLAN`. The plan records resolved volumes, worker mounts, S3 routes, claim names, namespace, and work directory. It contains no credentials. Existing Jobs with version 4 plans remain compatible. Do not set the plan environment variable or annotation in templates. + +If the annotation is missing, cleanup recovers the plan from the Job's worker environment. It validates the task and namespace before using that copy. Conflicting copies, invalid plans, or a storage-finalized Job with neither copy cause an error and retain the finalizer. Restore the correct plan before retrying cleanup in those cases. + +Cleanup validates resource ownership separately from S3 transfer routes. Version 4 plans with write-route overlaps rejected by newer workers can still be cleaned up. New submissions and workers reject those overlaps. + +The broader native storage migration changes the config schema. Migrate configurations to native objects and rebuild Go RPC plugins against the updated config package before exchanging configuration with them. This PVC naming change adds no config fields and requires no plugin rebuild by itself. The schema preserves `Kubernetes.ForbiddenPathPrefixes` at field 20 and uses `WorkerImage` at 24 and `Storage` at 25. It reserves `KubernetesStorage` field 2 and the names `Bindings` and `MountedInputs`. `MountedRead` and `MountedWrite` reserve field 1 and the name `WorkerMount`. Kubernetes field numbers 21–23 and the names `PVCMode`, `StorageClassName`, `SharedPVCName`, and `S3FilesFilesystemId` are also reserved. Configuration with retired names fails parsing. + +For configurations from PR #1454, replace `PVCMode: full` with native PV and PVC templates, `PVCMode: pvc` with a PVC template containing `spec.storageClassName`, and `PVCMode: shared` with `persistentVolumeClaim.claimName`. Keep `WorkerImage` as an independent field. + +Chicago's `S3FilesFilesystemId` used field 20, which conflicts with upstream `ForbiddenPathPrefixes`. Do not decode old Chicago binary configuration with the upstream schema. Both fields use a length-delimited wire representation, so a decoder cannot distinguish them. Export the configuration with its original schema, move the filesystem ID into the deployment's native CSI PV template, and load the migrated named configuration. Rebuild plugins against the canonical schema before exchanging configuration. Funnel supplies no automatic decoder for the conflicting binary format. + +The version 4 decoder cannot run or clean up Jobs with version 1, 2, or 3 plans. Finish those tasks and clean up their Jobs with a compatible older binary before upgrading. Jobs without a storage plan still require the legacy cleanup configuration during migration. These requirements apply whether the task uses existing claims or native PVC templates. + +Helm resolves deployment values before Funnel renders task variables. Charts must preserve literal Funnel expressions such as `{{.TaskId}}` through Helm rendering, as they do for existing Funnel Job templates. + +## Verify against a local cluster + +The opt-in integration test uses an existing Kubernetes context, creates temporary namespaces, and runs worker and executor Jobs with `alpine:3.21`. Use a disposable single-node cluster with a `standard` StorageClass and access to that image. The test checks file exchange through a provisioned claim, an existing claim, and an explicit hostPath PV and PVC pair. It then verifies ownership-based cleanup, including PV protection and Job finalizers. + +```sh +FUNNEL_KUBERNETES_TEST_KUBECONFIG=/path/to/test-kubeconfig \ + go test ./internal/compute/kubernetes -run '^TestNativeStorageLive$' -v -count=1 +``` + +The test verifies real Kubernetes mounts and cleanup. It does not certify an S3 CSI driver or asynchronous S3 output publication. + +To verify permission failures, run the RBAC test with an administrator kubeconfig for a disposable cluster. It creates temporary service accounts and roles, impersonates those accounts, and checks that denied PV permissions leave no worker Job or storage resources. It also checks successful submission and cleanup with PV permissions restricted by name and with a PVC-only configuration. + +```sh +FUNNEL_KUBERNETES_TEST_KUBECONFIG=/path/to/test-kubeconfig \ + go test ./internal/compute/kubernetes -run '^TestNativeStoragePermissionsLive$' -v -count=1 +``` diff --git a/docs/tools/funnel/docs/compute/kubernetes-templates/README.md b/docs/tools/funnel/docs/compute/kubernetes-templates/README.md new file mode 100644 index 00000000..3531f824 --- /dev/null +++ b/docs/tools/funnel/docs/compute/kubernetes-templates/README.md @@ -0,0 +1,11 @@ +# Kubernetes reference templates + +These files are historical reference material. Funnel does not embed them or use them as runtime defaults. Configure Kubernetes templates through the deployment's Helm chart. + +The reference files originated in the [Funnel Helm chart](https://github.com/ohsu-comp-bio/helm-charts/tree/funnel-0.1.60/charts/funnel/files). + +Templates used by Go tests live beside their tests: + +- [worker job](https://github.com/calypr/funnel/blob/738ab75a7056321d82cf57d9781d0af5a4433da6/internal/compute/kubernetes/testdata/worker-job.yaml) +- [service account](https://github.com/calypr/funnel/blob/738ab75a7056321d82cf57d9781d0af5a4433da6/internal/compute/kubernetes/resources/testdata/serviceaccount.yaml) +- [executor job](https://github.com/calypr/funnel/blob/738ab75a7056321d82cf57d9781d0af5a4433da6/internal/worker/testdata/executor-job.yaml) diff --git a/docs/tools/funnel/docs/compute/kubernetes-templates/role.yaml b/docs/tools/funnel/docs/compute/kubernetes-templates/role.yaml new file mode 100644 index 00000000..fce7d3b0 --- /dev/null +++ b/docs/tools/funnel/docs/compute/kubernetes-templates/role.yaml @@ -0,0 +1,39 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: funnel-worker-sa-{{.Namespace}}-{{.TaskId}}-role + namespace: {{.Namespace}} + labels: + app: funnel + taskId: {{.TaskId}} +rules: +# Job management permissions +- apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["create", "get", "list", "watch", "update", "patch", "delete"] +- apiGroups: ["batch"] + resources: ["jobs/status"] + verbs: ["get", "update", "patch"] + +# Pod management permissions (jobs create pods) +- apiGroups: [""] + resources: ["pods"] + verbs: ["create", "get", "list", "watch", "delete"] +- apiGroups: [""] + resources: ["pods/status", "pods/log"] + verbs: ["get", "list", "watch"] + +# ConfigMap and Secret access (if needed for job configuration) +- apiGroups: [""] + resources: ["configmaps", "secrets"] + verbs: ["get", "list"] + +# PVC management (if using persistent volumes) +- apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["create", "get", "list", "watch", "delete"] + +# Events (for job status reporting) +- apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch"] diff --git a/docs/tools/funnel/docs/compute/kubernetes-templates/rolebinding.yaml b/docs/tools/funnel/docs/compute/kubernetes-templates/rolebinding.yaml new file mode 100644 index 00000000..72ed8ae5 --- /dev/null +++ b/docs/tools/funnel/docs/compute/kubernetes-templates/rolebinding.yaml @@ -0,0 +1,16 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: funnel-worker-sa-{{.Namespace}}-{{.TaskId}}-binding + namespace: {{.Namespace}} + labels: + app: funnel + taskId: {{.TaskId}} +subjects: +- kind: ServiceAccount + name: funnel-worker-sa-{{.Namespace}}-{{.TaskId}} + namespace: {{.Namespace}} +roleRef: + kind: Role + name: funnel-worker-sa-{{.Namespace}}-{{.TaskId}}-role + apiGroup: rbac.authorization.k8s.io diff --git a/docs/tools/funnel/docs/compute/kubernetes-templates/worker-configmap.yaml b/docs/tools/funnel/docs/compute/kubernetes-templates/worker-configmap.yaml new file mode 100644 index 00000000..76654bec --- /dev/null +++ b/docs/tools/funnel/docs/compute/kubernetes-templates/worker-configmap.yaml @@ -0,0 +1,29 @@ +# Per-task worker ConfigMap template. +# +# This template is rendered once per task and should be used ONLY when the +# WorkerTemplate references a per-task ConfigMap name such as: +# name: funnel-worker-config-{{.TaskId}} +# +# For deployments that mount a shared static ConfigMap (e.g. "funnel-config") +# the same for every task, leave ConfigMapTemplate empty in the Funnel config. +# That avoids creating one ConfigMap per task in etcd, which would: +# - Duplicate the full config (including storage credentials) N times +# - Require configmaps RBAC (get/create/delete) on the funnel ServiceAccount +# - Add unnecessary API-server and etcd pressure at scale +# +# Available template variables: +# {{.TaskId}} — unique task identifier +# {{.Namespace}} — Kubernetes namespace for the job (JobsNamespace) +# {{.Config}} — full Funnel worker config serialized to YAML +# +apiVersion: v1 +kind: ConfigMap +metadata: + name: funnel-worker-config-{{.TaskId}} + namespace: {{.Namespace}} + labels: + app: funnel + taskId: {{.TaskId}} +data: + funnel-worker.yaml: | + {{.Config}} diff --git a/docs/tools/funnel/docs/compute/kubernetes-templates/worker-pv.yaml b/docs/tools/funnel/docs/compute/kubernetes-templates/worker-pv.yaml new file mode 100644 index 00000000..811e061d --- /dev/null +++ b/docs/tools/funnel/docs/compute/kubernetes-templates/worker-pv.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: funnel-worker-pv-{{.TaskId}} + labels: + app: funnel + taskId: {{.TaskId}} + namespace: {{.Namespace}} +spec: + storageClassName: "" # Required for static provisioning + capacity: + storage: {{if ne .DiskGb 0.0}}"{{printf "%.0fGi" .DiskGb}}"{{else}}"1Gi"{{end}} + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + mountOptions: + - allow-delete + - allow-overwrite + - region={{.Region}} + - file-mode=0755 + {{- if .KmsKeyID }} + - sse aws:kms + - sse-kms-key-id={{.KmsKeyID}} + {{- end }} + csi: + driver: s3.csi.aws.com + volumeHandle: s3-csi-{{.TaskId}} + volumeAttributes: + bucketName: {{.Bucket}} + claimRef: + namespace: {{.Namespace}} + name: funnel-worker-pvc-{{.TaskId}} diff --git a/docs/tools/funnel/docs/compute/kubernetes-templates/worker-pvc.yaml b/docs/tools/funnel/docs/compute/kubernetes-templates/worker-pvc.yaml new file mode 100644 index 00000000..46ef1552 --- /dev/null +++ b/docs/tools/funnel/docs/compute/kubernetes-templates/worker-pvc.yaml @@ -0,0 +1,17 @@ +# Worker/Executor PVC +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: funnel-worker-pvc-{{.TaskId}} + namespace: {{ .Namespace }} + labels: + app: funnel + taskId: {{.TaskId}} +spec: + storageClassName: "" # Required for static provisioning + accessModes: + - ReadWriteMany + resources: + requests: + storage: {{if ne .DiskGb 0.0}}"{{printf "%.0fGi" .DiskGb}}"{{else}}"1Gi"{{end}} + volumeName: funnel-worker-pv-{{.TaskId}} diff --git a/docs/tools/funnel/docs/compute/kubernetes.md b/docs/tools/funnel/docs/compute/kubernetes.md index a99d80b2..b0a99f49 100644 --- a/docs/tools/funnel/docs/compute/kubernetes.md +++ b/docs/tools/funnel/docs/compute/kubernetes.md @@ -1,20 +1,10 @@ ---- -title: Kubernetes -menu: - main: - parent: Compute - weight: 20 ---- - -!!! warning - Funnel on Kubernetes is in active development and may involve frequent updates 🚧 - # Kubernetes +> **Warning:** Funnel on Kubernetes is in active development and may involve frequent updates 🚧 + ## 1. Add Helm Repo -!!! note - See Funnel's [Helm Charts](https://github.com/calypr/helm-charts/tree/main/charts/funnel) for the latest configuration options +> **Note:** See Funnel's [Helm Charts](https://github.com/calypr/helm-charts/tree/main/charts/funnel) for the latest configuration options ```sh helm repo add calypr https://calypr.org/helm-charts @@ -59,6 +49,8 @@ funnel task get --view MINIMAL # Storage Architecture +The following diagram shows the legacy per-task S3 volume flow. For native storage bindings and other volume options, see the [Kubernetes storage guide](kubernetes-storage.md). + Funnel uses the [AWS S3 CSI Driver](https://github.com/awslabs/mountpoint-s3-csi-driver) to provision a per-task PersistentVolume (PV) and PersistentVolumeClaim (PVC) backed by S3. This gives Worker and Executor pods `ReadWriteMany` access across nodes without requiring them to co-schedule. ```mermaid @@ -82,11 +74,39 @@ sequenceDiagram Worker->>Server: Report COMPLETE ``` +# Startup cleanup + +When the internal reconciler is enabled, backlog cleanup runs in the background +so Kubernetes cleanup does not delay server startup. Periodic reconciliation +starts after that cleanup finishes; the two operations do not run concurrently. +Cleanup uses the server context, so stopping the server cancels in-flight cleanup +requests. + +`Kubernetes.DisableJobCleanup` skips backlog cleanup while keeping reconciliation +enabled. `Kubernetes.DisableReconciler` disables both automatic startup cleanup +and periodic reconciliation. + +# Executor image pull policy + +Executor containers use `Always` by default. Set the task tag `_IMAGE_PULL_POLICY` +to `Always`, `IfNotPresent`, or `Never` to override the policy for that task: + +```json +"tags": { + "_IMAGE_PULL_POLICY": "IfNotPresent" +} +``` + +An absent or empty tag keeps the default. Invalid values fail executor setup. +The tag applies to executor containers; it does not change the worker image's +pull policy. Custom executor templates must use `{{.ImagePullPolicy}}` in their +`imagePullPolicy` field to honor the tag. + # Additional Resources 📚 - [Helm Repo](https://calypr.org/helm-charts) -- [Helm Repo Source](https://github.com/ohsu-comp-bio/helm-charts) -- [Helm Chart values reference](https://github.com/ohsu-comp-bio/helm-charts/tree/main/charts/funnel) +- [Helm Repo Source](https://github.com/calypr/helm-charts) +- [Helm Chart values reference](https://github.com/calypr/helm-charts/tree/main/charts/funnel) - [AWS S3 CSI Driver](https://github.com/awslabs/mountpoint-s3-csi-driver) - [The Chart Best Practices Guide](https://helm.sh/docs/chart_best_practices/) - [GitHub issue #1412 — S3 Working Directory Behavior](https://github.com/calypr/funnel/issues/1412) diff --git a/docs/tools/funnel/docs/compute/pbs-torque.md b/docs/tools/funnel/docs/compute/pbs-torque.md index 59b2dfa1..76b92619 100644 --- a/docs/tools/funnel/docs/compute/pbs-torque.md +++ b/docs/tools/funnel/docs/compute/pbs-torque.md @@ -1,11 +1,3 @@ ---- -title: PBS/Torque -render_macros: true -menu: - main: - parent: Compute - weight: 20 ---- # PBS/Torque Funnel can be configured to submit workers to [PBS/Torque][pbs] by making calls @@ -15,32 +7,17 @@ The Funnel server needs to run on a submission node. Configure Funnel to use PBS by including the following config: It is recommended to update the submit file template so that the -`funnel worker run` command takes a config file as an argument -(e.g. `funnel worker run --config /opt/funnel_config.yml --taskID {% raw %}{{.TaskId}}{% endraw %}`) +`funnel worker run` command takes a config file as an argument +(e.g. `funnel worker run --config /opt/funnel_config.yml --taskID {{.TaskId}}`) -{% raw %} -```YAML -Compute: pbs +Download the [PBS/Torque submit template](https://github.com/calypr/funnel/blob/738ab75a7056321d82cf57d9781d0af5a4433da6/website/static/funnel-config-examples/pbs-template.txt) +to `/etc/funnel/pbs-template.txt`, then configure Funnel to read it: +```yaml +Compute: pbs PBS: - Template: | - #!/bin/bash - #PBS -N {{.TaskId}} - #PBS -o {{.WorkDir}}/funnel-stdout - #PBS -e {{.WorkDir}}/funnel-stderr - {{if ne .Cpus 0 -}} - {{printf "#PBS -l nodes=1:ppn=%d" .Cpus}} - {{- end}} - {{if ne .RamGb 0.0 -}} - {{printf "#PBS -l mem=%.0fgb" .RamGb}} - {{- end}} - {{if ne .DiskGb 0.0 -}} - {{printf "#PBS -l file=%.0fgb" .DiskGb}} - {{- end}} - - funnel worker run --taskID {{.TaskId}} + TemplateFile: /etc/funnel/pbs-template.txt ``` -{% endraw %} The following variables are available for use in the template: | Variable | Description | @@ -54,4 +31,4 @@ The following variables are available for use in the template: See https://golang.org/pkg/text/template for information on creating templates. -[pbs]: https://hpc-wiki.info/hpc/Torque +[pbs]: http://www.adaptivecomputing.com/products/open-source/torque/ diff --git a/docs/tools/funnel/docs/compute/slurm.md b/docs/tools/funnel/docs/compute/slurm.md index 98f16977..34e8e84d 100644 --- a/docs/tools/funnel/docs/compute/slurm.md +++ b/docs/tools/funnel/docs/compute/slurm.md @@ -1,46 +1,56 @@ ---- -title: Slurm -menu: - main: - parent: Compute - weight: 20 ---- # Slurm Funnel can be configured to submit workers to [Slurm][slurm] by making calls to `sbatch`. The Funnel server needs to run on a submission node. -Configure Funnel to use Slurm by including the following config: + +## Files and connectivity + +For each task, the server creates a directory under `Worker.WorkDir` and writes +a Slurm submission script there. The default script also puts the Slurm stdout +and stderr files in that task directory. Use an absolute `Worker.WorkDir` path +that is accessible and writable at the same location on the submission host +and the compute nodes, for example a shared filesystem: + +```yaml +Worker: + WorkDir: /shared/funnel/work +``` + +Use the same setting in the worker's configuration. The `funnel` executable, +the worker configuration file, and the container runtime must also be available +on compute nodes. If the worker uses an embedded server database such as BoltDB +or Badger, it reads tasks and reports events over RPC: configure +`RPCClient.ServerAddress` to an address reachable from the compute nodes. +For other database backends, the worker must be able to reach the configured +database and storage services. + +The shared working directory does not mean every input or output must be on +the shared filesystem. The worker stages task inputs from their URLs, runs the +container, and transfers declared outputs to their URLs. For `file://` URLs, +the referenced paths must be accessible on the compute node and permitted by +`LocalStorage.AllowedDirs`; see [local storage](../storage/local.md). + +Workers normally remove their task working directory when they finish. Failed +submissions can leave directories behind; check them before cleaning them up. +If you customize the Slurm template, ensure its log paths and any config path +passed to the worker are accessible from the compute node. + +## Slurm configuration It is recommended to update the submit file template so that the -`funnel worker run` command takes a config file as an argument -(e.g. `funnel worker run --config /opt/funnel_config.yml --taskID {% raw %}{{.TaskId}}{% endraw %}`) +`funnel worker run` command takes a config file as an argument +(e.g. `funnel worker run --config /opt/funnel_config.yml --taskID {{.TaskId}}`) -{% raw %} -```YAML -Compute: slurm +Download the [Slurm submit template](https://github.com/calypr/funnel/blob/738ab75a7056321d82cf57d9781d0af5a4433da6/website/static/funnel-config-examples/slurm-template.txt) +to `/etc/funnel/slurm-template.txt`, then configure Funnel to read it: +```yaml +Compute: slurm Slurm: - Template: | - #!/bin/bash - #SBATCH --job-name {{.TaskId}} - #SBATCH --ntasks 1 - #SBATCH --error {{.WorkDir}}/funnel-stderr - #SBATCH --output {{.WorkDir}}/funnel-stdout - {{if ne .Cpus 0 -}} - {{printf "#SBATCH --cpus-per-task %d" .Cpus}} - {{- end}} - {{if ne .RamGb 0.0 -}} - {{printf "#SBATCH --mem %.0fGB" .RamGb}} - {{- end}} - {{if ne .DiskGb 0.0 -}} - {{printf "#SBATCH --tmp %.0fGB" .DiskGb}} - {{- end}} - - funnel worker run --taskID {{.TaskId}} + TemplateFile: /etc/funnel/slurm-template.txt ``` -{% endraw %} The following variables are available for use in the template: | Variable | Description | diff --git a/docs/tools/funnel/docs/databases.md b/docs/tools/funnel/docs/databases.md new file mode 100644 index 00000000..2dbb9552 --- /dev/null +++ b/docs/tools/funnel/docs/databases.md @@ -0,0 +1 @@ +# Databases diff --git a/docs/tools/funnel/docs/databases/badger.md b/docs/tools/funnel/docs/databases/badger.md new file mode 100644 index 00000000..550027ce --- /dev/null +++ b/docs/tools/funnel/docs/databases/badger.md @@ -0,0 +1,33 @@ +# Badger + +[Badger](https://github.com/dgraph-io/badger) is an embedded database that stores +Funnel task data in a local directory. It does not require a separate database +server. Funnel uses BoltDB by default; select Badger in your configuration: + +```yaml +Database: badger + +# The selected database already stores task events. This avoids opening +# another database as an additional event writer. +EventWriters: + - log + +Badger: + Path: ./funnel-work-dir/funnel.badger.db +``` + +Save this configuration to a file, then start Funnel with it: + +```sh +funnel server run --config /path/to/config.yaml +``` + +`Badger.Path` is a directory, relative to the server's working directory when +given a relative path. Funnel creates it if it does not exist. Place it on +persistent storage if task data must survive server or container restarts. To +back it up, stop Funnel and copy the entire directory; restore it before +restarting the server. + +Use one Funnel server process per Badger directory. Badger does not implement +the scheduler service required by Funnel's `manual` compute backend; use a +different database backend for that configuration. diff --git a/docs/tools/funnel/docs/databases/boltdb.md b/docs/tools/funnel/docs/databases/boltdb.md index ea5885e0..44db2e5d 100644 --- a/docs/tools/funnel/docs/databases/boltdb.md +++ b/docs/tools/funnel/docs/databases/boltdb.md @@ -1,15 +1,7 @@ ---- -title: Embedded -menu: - main: - parent: Databases - weight: -10 ---- - # Embedded -By default, Funnel uses an embedded database named [BoltDB][bolt] to store task -and scheduler data. This is great for development and a simple server without +By default, Funnel uses an embedded database named [BoltDB][bolt] to store task +and scheduler data. This is great for development and a simple server without external dependencies, but it doesn't scale well to larger clusters. Available config: diff --git a/docs/tools/funnel/docs/databases/datastore.md b/docs/tools/funnel/docs/databases/datastore.md index ea31d8c6..1fd86aad 100644 --- a/docs/tools/funnel/docs/databases/datastore.md +++ b/docs/tools/funnel/docs/databases/datastore.md @@ -1,10 +1,3 @@ ---- -title: Datastore -menu: - main: - parent: Databases ---- - # Google Cloud Datastore Funnel supports storing tasks (but not scheduler data) in Google Cloud Datastore. @@ -38,7 +31,7 @@ to support the task-list queries. This is typically done through command-line by referencing an **index.yaml** file (do not change the filename) with the following content: -```shell +```sh gcloud datastore indexes create path/to/index.yaml --database='funnel' ``` diff --git a/docs/tools/funnel/docs/databases/dynamodb.md b/docs/tools/funnel/docs/databases/dynamodb.md index 3e536c21..0e27f338 100644 --- a/docs/tools/funnel/docs/databases/dynamodb.md +++ b/docs/tools/funnel/docs/databases/dynamodb.md @@ -1,10 +1,3 @@ ---- -title: DynamoDB -menu: - main: - parent: Databases ---- - # DynamoDB Funnel supports storing task data in DynamoDB. Storing scheduler data is not supported currently, so using the node scheduler with DynamoDB won't work. Using AWS Batch for compute scheduling may be a better option. diff --git a/docs/tools/funnel/docs/databases/elasticsearch.md b/docs/tools/funnel/docs/databases/elasticsearch.md deleted file mode 100644 index e397348a..00000000 --- a/docs/tools/funnel/docs/databases/elasticsearch.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -title: Elasticsearch -menu: - main: - parent: Databases ---- - -# Elasticsearch - -Funnel supports storing tasks and scheduler data in Elasticsearch (v8). - -Config: -```yaml -Database: elastic - -Elastic: - # Prefix to use for indexes - IndexPrefix: "funnel" - URL: http://localhost:9200 - # Optional. Username for HTTP Basic Authentication. - Username: - # Optional. Password for HTTP Basic Authentication. - Password: - # Optional. Endpoint for the Elastic Service (https://elastic.co/cloud). - CloudID: - # Optional. Base64-encoded token for authorization; if set, overrides username/password and service token. - APIKey: - # Optional. Service token for authorization; if set, overrides username/password. - ServiceToken: -``` diff --git a/docs/tools/funnel/docs/databases/mongodb.md b/docs/tools/funnel/docs/databases/mongodb.md index 4a6e8abc..a4ece7c3 100644 --- a/docs/tools/funnel/docs/databases/mongodb.md +++ b/docs/tools/funnel/docs/databases/mongodb.md @@ -1,24 +1,151 @@ ---- -title: MongoDB -menu: - main: - parent: Databases ---- - # MongoDB -Funnel supports storing tasks and scheduler data in MongoDB. +Funnel supports storing tasks and scheduler data in MongoDB. A replica set or +sharded cluster is required. Standalone MongoDB servers are rejected at startup +because task assignment and resource release update multiple documents atomically. +A single-node replica set is sufficient for development; it provides no redundancy. + +## Connect Funnel to a replica set + +Use the replica-set name and member addresses from your deployment. -Config: ```yaml Database: mongodb MongoDB: - # Addresses for the seed servers. + # Replica-set seed addresses, or one MongoDB connection URI. Addrs: - - "localhost" + - "mongodb://mongo-1:27017,mongo-2:27017,mongo-3:27017/?replicaSet=rs0" # Database name used within MongoDB to store funnel data. Database: "funnel" Username: "" Password: "" ``` + +For production, use a replica set with redundant members and authentication. +Follow MongoDB's [replica-set deployment guide](https://www.mongodb.com/docs/manual/tutorial/deploy-replica-set/) +for member networking and authentication. Each advertised member hostname must +be reachable from Funnel. Keep `directConnection` unset so the client can discover +the current primary. + +Set `Username` and `Password` separately, or include credentials in the URI. +When using a URI, specify `authSource` if the user belongs to a different +authentication database. Separate credentials take precedence over credentials +in the URI. Percent-encode credentials included in a URI. + +To trust a custom certificate authority, make the CA file available to the +Funnel process and set `tls=true` and `tlsCAFile` in the connection URI: + +```yaml +MongoDB: + Addrs: + - "mongodb://mongo.example:27017/?replicaSet=rs0&authSource=admin&tls=true&tlsCAFile=%2Fetc%2Ffunnel%2Fmongo-ca.pem" +``` + +For mutual TLS, the MongoDB driver also accepts `tlsCertificateKeyFile` in the +URI. Its PEM file must contain the client certificate and private key. + +## Start a single-node replica set locally + +This Docker example runs MongoDB without authentication and publishes its port +only on the host's loopback address. Run Funnel on the same host. The named volume +preserves MongoDB data and replica-set configuration across container restarts. + +1. Start MongoDB with replica-set support: + + ```sh + docker run -d --name funnel-mongo \ + -p 127.0.0.1:27017:27017 \ + -v funnel-mongo-data:/data/db \ + mongo:7 --replSet rs0 --bind_ip_all + ``` + +2. Check that MongoDB accepts connections: + + ```sh + docker exec funnel-mongo mongosh --quiet --eval 'db.runCommand({ping: 1})' + ``` + + Wait for `{ ok: 1 }` before continuing. If the connection fails during startup, + repeat the command. + +3. Initialize the replica set once for this data volume: + + ```sh + docker exec funnel-mongo mongosh --quiet --eval \ + 'rs.initiate({_id: "rs0", members: [{_id: 0, host: "localhost:27017"}]})' + ``` + + Check that the node has become primary: + + ```sh + docker exec funnel-mongo mongosh --quiet --eval 'db.hello().isWritablePrimary' + ``` + + Wait for `true` before starting Funnel. Initialization can return before the + primary election finishes. Repeat the check if it returns `false`. + +4. Set Funnel's database configuration: + + ```yaml + Database: mongodb + MongoDB: + Addrs: + - "mongodb://localhost:27017/?replicaSet=rs0&directConnection=true" + Database: funnel + ``` + + `directConnection=true` keeps the client on the published Docker port. This + example advertises `localhost` inside the container and is intended for local + development. If Funnel also runs in a container, use an address reachable + from that container instead of its own `localhost`. + +To restart this container, run `docker start funnel-mongo`. Do not repeat +`rs.initiate()` for an already initialized volume. This single-node setup supports +transactions but provides no redundancy. + +## Upgrade an existing standalone server + +Convert the existing server before starting the upgraded Funnel version: + +1. Stop the older Funnel servers and back up MongoDB. +2. Stop MongoDB and add the following to its existing `mongod.conf`, preserving + its data directory and other settings: + + ```yaml + replication: + replSetName: rs0 + ``` + +3. Configure replica-set member authentication if your server uses authentication. + Follow MongoDB's [standalone conversion guide](https://www.mongodb.com/docs/manual/tutorial/convert-standalone-to-replica-set/) + for keyfile or X.509 setup. Preserve the existing users and data. +4. Restart MongoDB and initialize the replica set through `mongosh` with + `rs.initiate()`. Check `rs.conf()` and ensure the advertised member address is + reachable from Funnel. Wait for `db.hello().isWritablePrimary` to return `true`. +5. Update Funnel's connection URI to include `replicaSet=rs0`, then start the + upgraded Funnel server. Use your actual replica-set name if it differs. + +Changing the connection URI alone does not enable transactions. The MongoDB +server must be configured and initialized as a replica set. + +## Funnel's startup migration + +When Funnel initializes the MongoDB backend, it ensures a unique `id` index +with simple collation on the existing `nodes` collection. If legacy node +documents contain duplicate IDs, initialization fails and reports that the +duplicates must be resolved before startup. The index migration does not +delete node documents or remove existing indexes. + +Funnel also creates a unique task-ID index, a queue index, and an index of active +allocations. At the first upgraded startup it imports active node assignments, +snapshots their reserved resources, and stores heartbeat timestamps separately. +Conflicting task owners abort the migration without committing partial claims. +Stop older Funnel servers before upgrading, and resolve reported conflicts +before restarting submission. Back up the database before upgrading. + +Assignment commits the task owner and node capacity together. Terminal state +updates release the original reservation in the same transaction, and repeated +completion events cannot release it twice. Ordinary heartbeats do not rewrite +resource reservations. The scheduler uses the same assignment protocol as +BoltDB and PostgreSQL. diff --git a/docs/tools/funnel/docs/databases/postgres.md b/docs/tools/funnel/docs/databases/postgres.md new file mode 100644 index 00000000..1dd89a82 --- /dev/null +++ b/docs/tools/funnel/docs/databases/postgres.md @@ -0,0 +1,35 @@ +# Postgres + +> [!WARNING] +> +> Postgres support is in early development. +> +> Production deployments are recommended to use the alternative storage backends available, such as MongoDB and DynamoDB. + +## Configuration + +```yaml +Database: postgres + +Postgres: + Host: localhost + Database: funnel + User: funnel + Password: example +``` + +## Default Values + +```go + Postgres: &Postgres{ + Host: "localhost:5432", + Database: "funnel", + User: "funnel", + Password: "example", + Timeout: &TimeoutConfig{ + TimeoutOption: &TimeoutConfig_Duration{ + Duration: durationpb.New(time.Second * 30), + }, + }, + }, +``` diff --git a/docs/tools/funnel/docs/development.md b/docs/tools/funnel/docs/development.md new file mode 100644 index 00000000..40b99510 --- /dev/null +++ b/docs/tools/funnel/docs/development.md @@ -0,0 +1,4 @@ +# Development + +See [Continuous integration](development/ci.md) for test ownership, reports, +and failure diagnostics. diff --git a/docs/tools/funnel/docs/development/ci.md b/docs/tools/funnel/docs/development/ci.md new file mode 100644 index 00000000..1b9928f8 --- /dev/null +++ b/docs/tools/funnel/docs/development/ci.md @@ -0,0 +1,65 @@ +# Continuous integration + +PR checks separate test environments while keeping database startup out of the +general Go race suite's shared Docker workload. + +| Check | Coverage | +| --- | --- | +| Go tests | All Go packages except the four packages assigned to the scheduler matrix; race detector, coverage, two packages at a time | +| Scheduler tests: BoltDB | Scheduler contract against BoltDB | +| Scheduler tests: PostgreSQL | PostgreSQL package tests, scheduler contract, and independent process/crash recovery tests, sequentially | +| Scheduler tests: MongoDB | MongoDB package tests, scheduler contract, and independent process/crash recovery tests, sequentially | +| Docker worker integration | Resource limits, cancellation, execution admission, and task output contracts with the Docker integration gate enabled | +| Local storage executor permissions | The executor ownership test under root, which creates and tests distinct user identities | +| Public API | A separate Go module consuming the TES client and plugin SDK | +| Dashboard | JavaScript tests, production bundle consistency, and browser workflows against Funnel | +| Kubernetes storage | PVC/S3 storage and failure recovery in k3d | +| Nextflow | Local execution and Kubernetes/S3 execution in separate environments | +| Gen3 | Authentication, authorization, storage, and workflows through the Gen3 stack | +| Compliance | External TES 1.0 and 1.1 compliance suites from a pinned runner revision | + +Static analysis, vulnerability checks, website builds, and multi-platform image +builds provide additional checks. The image publishing workflow and compliance +workflow run on pushes; most test workflows run on PRs and pushes to `develop`. +Some workflows have narrower branch or path filters; their triggers are defined +under `.github/workflows`. + +The four Go packages assigned outside the general suite are +`internal/database/postgres`, `internal/database/mongodb`, +`tests/integration/schedulercontract`, and `tests/e2e/schedulerprocess`. +Their JUnit reports and coverage profiles are uploaded by the scheduler matrix. +The general suite uploads its own report and coverage profile. + +The `unitTest` and `Integration tests / integration_tests` jobs preserve existing +required status names. They aggregate results and do not run tests again. + +## Startup and failure handling + +Database and worker images are pulled before tests start. Image downloads have +bounded retries; assertions and complete test runs are not automatically retried. +Database readiness checks remain in place, and startup deadlines are separate +from the database operation deadlines. Keep the pre-pulled Ryuk version aligned +with the version used by `testcontainers-go`. + +Failed Go jobs upload Docker daemon information, image inventory, and available +test-container state and logs. Full container inspection is deliberately omitted +because it includes credential environment variables. Test fixtures clean up +containers even when startup fails, so a removed container's logs may only be +available in the test output. + +The in-process test harness reserves its actual server sockets until startup and +releases them on cleanup. Node-drain tests wait for observed lifecycle states and +drive scheduler passes explicitly. PostgreSQL initialization also cancels stalled +connections when its initialization budget expires; a retry budget alone cannot +interrupt a blocked network operation. + +All CI jobs have explicit time limits. Network download retries are bounded, and +readiness probes fail when their wait budget expires. The reusable binary cache +is keyed by OS, architecture, and commit to include embedded assets and build +metadata in its identity. + +Run the CI helper regressions locally with GNU `timeout` available: + +```sh +python3 .github/scripts/test-ci-helpers.py +``` diff --git a/docs/tools/funnel/docs/development/developers.md b/docs/tools/funnel/docs/development/developers.md index c18a4ec1..cc80497d 100644 --- a/docs/tools/funnel/docs/development/developers.md +++ b/docs/tools/funnel/docs/development/developers.md @@ -1,12 +1,3 @@ ---- -title: Funnel Developers - -menu: - main: - parent: Development - weight: 30 ---- - # Developers This page contains a rough collection of notes for people wanting to build Funnel from source and/or edit the code. @@ -17,12 +8,12 @@ This page contains a rough collection of notes for people wanting to build Funne 2. Ensure GOPATH is set. See [the docs][gopath] for help. Also, you probably want to add `$GOPATH/bin` to your `PATH`. 3. Clone funnel and build - ```shell + ```sh $ git clone https://github.com/ohsu-comp-bio/funnel.git $ cd funnel $ make ``` - + 4. Funnel is now downloaded and installed. Try `funnel version`. 5. You can edit the code and run `make install` to recompile. @@ -36,7 +27,7 @@ A Funnel development environment includes: - [gRPC Gateway][gateway] for HTTP communication. - [Angular][angular] and [SASS][sass] for the web dashboard. - [GNU Make][make] for development tasks. -- [Docker](https://docker.io) for executing task containers (tested with v1.12, v1.13). +- [Docker][docker] for executing task containers (tested with v1.12, v1.13). - [dep][dep] for Go dependency vendoring. - [Make][make] for development/build commands. - [NodeJS][node] and [NPM][npm] for web dashboard development. @@ -45,11 +36,45 @@ A Funnel development environment includes: Most development tasks are run through `make` commands, including build, release, testing, website docs, lint, tidy, webdash dev, and more. See the [Makefile](https://github.com/ohsu-comp-bio/funnel/blob/master/Makefile) for an up-to-date list of commands. +### Error handling + +The Go static-analysis CI job runs `errcheck` for production code and tests, +including errors assigned to `_`. Handle errors that can affect results or +propagate them to the caller. Test setup and assertions must also check errors. + +An intentional exception must be local and name the linter, with a concrete +reason. For example, a read-only file close after a completed read may use: + +```go +_ = input.Close() //nolint:errcheck // Closing this read-only input cannot invalidate the bytes already read. +``` + +CI rejects unexplained or nonspecific suppressions. Do not add package-wide +exclusions, generic ignore-error helpers, or log-and-continue code merely to +silence the check. A failed write, upload, parse, or state update normally needs +handling. Expected rollback after commit and best-effort cleanup require an +explicit decision at the call site. + +### Unreachable Go code + +Run `make deadcode` for the same check used by CI. It runs a pinned analyzer for +Linux/amd64 with tests included. It analyzes test entry points without executing +tests. Generated functions are excluded by the analyzer's default settings. + +The existing findings are recorded in `.github/deadcode-baseline.txt`. New +findings fail CI for review. Unreachable code may represent unfinished wiring, +external API contracts, or obsolete functionality. The check never deletes code. + +Wire up intended behavior or remove confirmed obsolete code. If a finding is +intentional, explain it in the pull request before adding it to the baseline. +Run `make deadcode-update` to refresh the baseline and review the diff. CI also +rejects stale entries when findings disappear, so the baseline can shrink. + ### Go Tests -Run all tests: `make test` -Run the worker tests: `go test ./worker/...` -Run the worker tests with "Cancel" in the name: `go test ./worker -run Cancel` +Run all tests: `make test` +Run the worker tests: `go test ./internal/worker/...` +Run the worker tests with "Cancel" in the name: `go test ./internal/worker -run Cancel` You get the idea. See the `go test` docs for more. @@ -64,6 +89,8 @@ mock interfaces in test code, for example, to mock the Google Cloud APIs. [grpc]: http://www.grpc.io/ [sass]: http://sass-lang.com/ [make]: https://www.gnu.org/software/make/ +[docker]: https://docker.io +[python]: https://www.python.org/ [dep]: https://golang.github.io/dep/ [node]: https://nodejs.org [npm]: https://www.npmjs.com/ @@ -75,21 +102,32 @@ mock interfaces in test code, for example, to mock the Google Cloud APIs. ### Making a release -- Update Makefile, edit `FUNNEL_VERSION` and `LAST_PR_NUMBER` - - `LAST_PR_NUMBER` can be found by looking at the previous release notes - from the previous release. -- Run `make website`, which updates the download links and other content. - - Check the website locally by running `make website-dev` -- Commit these changes. - - Because goreleaser requires a clean working tree in git - - This is a special case where it's easiest to commit to master. -- Create a git tag: `git tag X.Y.Z` -- Run `make release` - - This will build cross-platform binaries, build release notes, - and draft an unpublished GitHub release. - - Check the built artifacts by downloading the tarballs from the GitHub draft release - and running `funnel version`. -- `git push origin master` to push your website and release changes. -- A tagged docker image for the release will be built automatically on [dockerhub](https://hub.docker.com/repository/docker/quay.io/ohsu-comp-bio/funnel). -- Publish the draft release on GitHub. -- Copy `build/release/funnel.rb` to the `ohsu-comp-bio/homebrew-formula/Formula/funnel.rb` Homebrew formula repo, and push those changes to master. +Releases are published by the [Publish Funnel release workflow](https://github.com/calypr/funnel/actions/workflows/release.yaml). +It runs when a `v*.*.*` tag is pushed, and also supports **Run workflow** or `gh workflow run`. +The workflow checks out the requested tag, rather than publishing the selected branch. + +1. Merge the reviewed PR stack into `develop` and confirm its CI passes. +2. Review `CHANGELOG.md` and [the upgrade guide](../upgrading.md). Include import migrations, configuration migrations, and the requirement to rebuild net/rpc plugins with the host. No Makefile version edit is needed: release versions come from tags. +3. Choose an unused version. Check both Git tags and GitHub releases: an existing tag without a release still names older code. Do not move it to new code. In this repository `v0.12.0` already exists. +4. Before tagging, run `make snapshot` or manually dispatch the **Release packaging** workflow on the candidate branch. Packaging is not a pull-request check. This builds archives and checksums without publishing. Extract the native archive into a temporary directory and check `funnel version`, `funnel --help`, and basic task execution. Check that `docs/upgrading.md` is included. Snapshot versions include a development suffix. +5. Create an RC tag on the reviewed commit, for example `git tag -a vNEXT-rc.1 -m 'Release candidate'`, replacing `NEXT` with the chosen numeric version. Pushing the tag triggers publication immediately. RC tags become GitHub prereleases. +6. To validate an existing tag without publishing, select **Run workflow**, enter its tag, and leave **dry_run** enabled. The archives are retained as workflow artifacts. From the CLI: + + ```sh + gh workflow run release.yaml --repo calypr/funnel --ref develop \ + -f tag=vNEXT-rc.1 -F dry_run=true + ``` + + The workflow must first be available on the repository's default branch for GitHub to offer manual dispatch. For a local tag that has not been pushed, use the local snapshot check instead. +7. Test the RC using `bash install.sh vNEXT-rc.1 /tmp/funnel-rc-bin` from a temporary working directory. Check the reported version and run a representative task with your deployment configuration. +8. Tag the validated commit with the final version and push that tag. The release is published automatically, not saved as a draft. If retrying publication for an existing tag manually, set **dry_run** to false. Inspect existing assets first; do not overwrite a published version with different code. + +The separate container workflow publishes multi-architecture images to +`quay.io/ohsu-comp-bio/funnel` on pushes. Stable version tags update `latest`; +RC tags do not. Verify the image workflow as well as the binary release workflow. +Manual binary publication does not rerun the container workflow. + +`make release` also publishes from a clean tagged checkout using `GITHUB_TOKEN`. +Prefer the GitHub workflow for publication. Release notes come from the reviewed `Unreleased` section of `CHANGELOG.md`. +Both local and CI packaging use the +GoReleaser version pinned in the Makefile and its explicit executable path. diff --git a/docs/tools/funnel/docs/development/diagnostics.md b/docs/tools/funnel/docs/development/diagnostics.md new file mode 100644 index 00000000..78c9a9d8 --- /dev/null +++ b/docs/tools/funnel/docs/development/diagnostics.md @@ -0,0 +1,32 @@ +# Troubleshooting information + +Run `funnel debug` to print the Funnel version and configuration as JSON. +Pass the configuration file and any server flags used by your deployment: + +```sh +funnel debug --config /path/to/config.yaml +``` + +Commands that load Funnel configuration also accept `FUNNEL_CONFIG` as the +default configuration file path and `FUNNEL_LOG_LEVEL` as a log level override: + +```sh +export FUNNEL_CONFIG=/path/to/config.yaml +export FUNNEL_LOG_LEVEL=debug +funnel debug +``` + +A nonempty `--config` overrides `FUNNEL_CONFIG`. Log level precedence, from +lowest to highest, is the built-in default, configuration file, +`FUNNEL_LOG_LEVEL`, and `--Logger.Level`. Empty environment values are ignored. +An unreadable configuration file selected through `FUNNEL_CONFIG` returns an error. +These variables apply to CLI configuration loading; the public Go schema and +default configuration functions do not read them. + +The command reads the local file, environment overrides, and flags. It does not query a running server, +so use the same configuration and flags as the process you are investigating. +The report redacts known credentials, URL user information, paths and query strings, +environment and metadata values, commands, and templates. It retains other +configuration settings, such as endpoints and paths. Arbitrary values can still +contain sensitive information, so review the report and any logs before +sharing them. The report does not include logs or stack traces. diff --git a/docs/tools/funnel/docs/development/go-packages.md b/docs/tools/funnel/docs/development/go-packages.md new file mode 100644 index 00000000..33b2a38e --- /dev/null +++ b/docs/tools/funnel/docs/development/go-packages.md @@ -0,0 +1,159 @@ +# Go packages and the TES client + +For existing consumers, follow the [Funnel upgrade guide](../upgrading.md) to +migrate imports, client credentials, and plugins. + +Funnel separates TES wire models from the REST client: + +- `github.com/calypr/funnel/api/tes`: shared generated TES models, enum conveniences, + and request filters. It has no HTTP client or server dependencies. +- `github.com/calypr/funnel/client/tes`: TES REST operations, client configuration, + HTTP responses, and task polling. + +Server implementations are under `internal/`, where Go prevents external modules +from importing them. + +```go +client, err := tesclient.NewClient("https://example.org/prefix", + tesclient.WithHTTPClient(&http.Client{Timeout: 30 * time.Second})) +if err != nil { + return err +} +client.Token = token +result, err := client.CreateTask(ctx, &tes.Task{ + Executors: []*tes.Executor{{ + Image: "alpine", + Command: []string{"echo", "hello"}, + }}, +}) +``` + +Import `github.com/calypr/funnel/client/tes` as `tesclient` and +`github.com/calypr/funnel/api/tes` for the models. Import `context`, `net/http`, and +`time` as needed. +The client preserves URL prefixes and accepts a caller-owned HTTP client for +transports, timeouts, proxies, and testing. List responses return the next page +token; callers choose when to fetch another page. Requests use the caller's +context, and the server owns task validation and initialization. + +`NewClient` reads `FUNNEL_SERVER_USER`, `FUNNEL_SERVER_PASSWORD`, +`FUNNEL_SERVER_TOKEN`, and `FUNNEL_ALLOW_INSECURE_CREDENTIALS` as constructor +defaults, preserving existing library and CLI behavior. Set `User`, `Password`, +or `Token` afterward, or use a `ClientOption`, to override those defaults. +Bearer tokens take precedence over Basic credentials. Credentials require HTTPS +except for loopback hosts. Remote plaintext credentials and authenticated +HTTPS-to-HTTP redirects require `FUNNEL_ALLOW_INSECURE_CREDENTIALS=true` or +`tesclient.WithInsecureCredentials()`. CLI commands use the same public constructor. + +The client and dashboard node requests share a focused `internal/httpauth` credential +transport policy. It is a private helper with only standard-library dependencies. +This policy covers plaintext restrictions and redirects; it +does not read process environment variables. HTTP response parsing remains +private to the TES client. The CLI owns output formatting, and callers that need +model JSON serialization can use protobuf's `protojson` package directly. + +The former `tes.NewClient` API moves to `client/tes.NewClient`; its request and +response types use `api/tes` imports. There is no forwarding client in the +model package. The mutable `tes.Marshaler` global, `tes.MarshalToString` wrapper, +and the client's formatting-only `Marshaler` field have been removed. + +The root `version` package exposes build metadata without importing application +implementation. Makefile and release builds populate it through linker flags. + +## Application structure + +- `internal/cli`: commands and application assembly; root `main.go` remains the + executable entry point, including `go install github.com/calypr/funnel`. +- `internal/server`: listener lifecycle, service registration, route assembly, and + graceful shutdown. It serves generated RPC interfaces rather than concrete task + service implementations. +- `internal/server/auth`: Basic/OIDC authentication, JWKS, audit, and event-write + authorization. +- `internal/server/middleware`: RPC recovery/debug logging and HTTP message/body + limits and plugin header forwarding. +- `internal/server/gateway`: JSON marshaling, task-view presentation, content + negotiation, and HTTP error responses. +- `internal/server/taskservice`: TES task operations, plugin actions, and compute/ + event coordination. Its Close waits for submissions and closes owned plugin + processes once. +- `internal/identity`: request identity and task-access policy shared by server + authentication and database implementations. +- `internal/compute`, `internal/database`, `internal/worker`, `internal/storage`, + `internal/events`, and `internal/metrics`: application subsystems. +- `internal/compute`: shared backend contracts and validation; HPC implementations + live under `internal/compute/hpc`. +- `internal/worker/workspace`: task path mapping, confinement, and mounted volumes. +- `internal/cli/gce`: the GCE command and its metadata-to-config adapter. +- `internal/task`: task validation, initialization, state transitions, + projections, log mutation, and server pagination policy. +- `internal/tesrpc`: generated gRPC and HTTP gateway service bindings. +- `internal/pluginmanager`: application plugin loading. +- `internal/config`: application defaults, YAML loading/writing, templates and + examples, redaction, and response-limit resolution. +- `internal/config/aws`: AWS SDK configuration loading shared by AWS consumers; + the parent config package does not import it. +- `internal/logger`: runtime logging, logging defaults, and server interceptors. +- `internal/storage`: storage enablement and credential/environment selection. +- `internal/compute/kubernetes/taskresources`: Kubernetes task defaults and limits. +- `internal/webdash`: Go dashboard serving and embedded build output. Frontend + source and npm commands remain in `webdash/`. + +The public `api/config` and `api/logger` packages contain generated schemas. +The net/rpc adapters in `api/plugin` carry complete protobuf messages inside Gob +to preserve optional fields and oneofs without global type registrations. +Server address formatting lives in `internal/config`, alongside application +configuration defaults and loading. `api/plugin` contains the public plugin contract, handshake, and net/rpc and +gRPC adapters. Host process discovery, launching, and logging stay in +`internal/pluginmanager`. The former `plugins/shared` package has been removed; +plugin implementations should import `api/plugin`. External Go consumers must update imports and rebuild plugins. Plugin configuration, TES messages, +protobuf field numbers, service names, and JSON names remain compatible. +Application implementation imports such as `funnel/server`, `funnel/worker`, +and TES server helpers are no longer public Go APIs. External applications should +use the REST client; application extensions must live inside this module or use +the plugin contracts. No forwarding packages preserve the former import paths. + +The shared integration harness lives in `tests/internal/testenv`. Go restricts its +imports to the tests tree; individual package fixtures remain beside their owners. +Directory creation uses `os.MkdirAll` directly, with `filepath.Dir` when creating +parents for a file. Existing regular files cannot satisfy directory requests. + +## Generation and verification + +Run `make proto` to generate public message types from the TES OpenAPI schema +and service bindings from the same source. The schemas share the protobuf +package `tes`; Go code uses separate public and internal packages. The REST +schema lives at `api/tes/tes.swagger.json`. All seven protobuf sources and the +Buf module, dependency lock, and generation configuration live under `api/`. +The TES-specific converter lives in `api/tes/internal/openapi2proto`. +`make proto` converts the upstream TES OpenAPI schema to protobuf, then runs +`buf generate` from `api/`. It installs pinned local Go plugins; +`api/buf.gen.yaml` controls messages, gRPC, gateway, and Swagger generation. +Go output follows `go_package`, keeping private bindings under top-level +`internal/`. There is one generation configuration; the former unused Python +generation template has been removed. + +Proto imports are relative to `api/`, preserving logical paths such as +`tes/tes.proto` and `config/config.proto`. The plugin schema moves from +`plugins/proto/plugin.proto` to `plugin/plugin.proto`, and the scheduler schema +moves from `internal/compute/scheduler/scheduler.proto` to +`internal/scheduler/scheduler.proto`. Downstream proto consumers must update +these imports. Protobuf namespaces and service identities stay unchanged; these +file moves intentionally break the configured Buf `FILE` compatibility policy. +Run Buf commands from `api/`, including `buf lint` and `buf breaking --against` +with the desired baseline. + +Run `python3 .github/scripts/check-public-api.py` to check the client dependency +boundary, compile an external client and plugin consumer, and verify Go rejects +external imports of server implementations. CI runs this check on pull requests. + +This remains one Go module. The client compiles without server, cloud, database, +Kubernetes, gRPC, or gateway packages, but independent client dependency versions +and releases would require a separate module. + +Configuration type imports remain `github.com/calypr/funnel/api/config` and +`github.com/calypr/funnel/api/logger`. Application helpers such as `DefaultConfig`, +`ParseFile`, `ToYamlFile`, and `Safe` are private to Funnel. Plugins should use +the configuration supplied by the host and protobuf serialization directly. +The public schemas have no dependency on YAML loaders, validators, Kubernetes, +or application internals. CI checks this and exercises plugin protobuf and Gob +serialization from a separate module. diff --git a/docs/tools/funnel/docs/docs.md b/docs/tools/funnel/docs/docs.md index cfc87e4d..9c382d05 100644 --- a/docs/tools/funnel/docs/docs.md +++ b/docs/tools/funnel/docs/docs.md @@ -1,38 +1,36 @@ ---- -title: Overview -menu: - main: - identifier: docs - weight: -1000 ---- - # Overview -Funnel makes distributed batch processing easier by providing a simple task API and a set of +Funnel makes distributed, batch processing easier by providing a simple task API and a set of components which can easily adapted to a vareity of platforms. +For an existing deployment or Go integration, follow the [Funnel upgrade guide](upgrading.md) +before moving to the unreleased package and configuration changes. + ### Task A task defines a unit of work: metadata, input files to download, a sequence of Docker containers + commands to run, output files to upload, state, and logs. The API allows you to create, get, list, and cancel tasks. -Tasks are accessed via the `funnel task` command. Additionally the HTTP API is based on a standarized GA4GH -protocol, and any client the is compatible with that API will work with Funnel. +Tasks are accessed via the `funnel task` command. Go applications use the +[REST client][clientpkg] and [TES wire models][tespkg]. -There's a lot more you can do with the task API. See the [tasks docs](/tools/funnel/docs/tasks/) for more. +There's a lot more you can do with the task API. See the [tasks docs](tasks.md) for more. ### Server The server serves the task API, web dashboard, and optionally runs a task scheduler. It serves both HTTP/JSON and gRPC/Protobuf. +Run the server with `funnel server`. Its implementation lives in `internal/server`. + ### Storage Storage provides access to file systems such as S3, Google Storage, and local filesystems. Tasks define locations where files should be downloaded from and uploaded to. Workers handle the downloading/uploading. -See the [storage docs](/tools/funnel/docs/storage/) for more information on configuring storage backends. +See the [storage docs](storage.md) for more information on configuring storage backends. +Storage implementations live in `internal/storage` and are private application code. ### Worker @@ -52,7 +50,8 @@ Along the way, the worker writes logs to event streams and databases: - a list of output files uploaded, with sizes - system logs, such as host name, docker command, system error messages, etc. -The worker is accessible via the `funnel worker` command. +Run a worker with `funnel worker`. Its runtime lives in `internal/worker`, with +task path mapping in `internal/worker/workspace`. ### Node Scheduler @@ -66,6 +65,10 @@ external system for scheduling tasks and managing cluster resources, such as AWS or HPC systems like HTCondor, Slurm, Grid Engine, etc. Funnel provides integration with these services that doesn't include nodes or scheduling by Funnel. -See [Deploying a cluster](/tools/funnel/docs/compute/deployment/) for more information about running a cluster of nodes. +See [Deploying a cluster](compute/deployment.md) for more information about running a cluster of nodes. + +Run a node with `funnel node`. Its implementation lives in `internal/compute/scheduler`. -The node is accessible via the `funnel node` command. +[tes]: https://github.com/ga4gh/task-execution-schemas +[clientpkg]: https://github.com/calypr/funnel/tree/feature/client-server-split/client/tes +[tespkg]: https://github.com/calypr/funnel/tree/feature/client-server-split/api/tes diff --git a/docs/tools/funnel/docs/download.md b/docs/tools/funnel/docs/download.md index b7acd691..1ade690d 100644 --- a/docs/tools/funnel/docs/download.md +++ b/docs/tools/funnel/docs/download.md @@ -1,17 +1,37 @@ ---- -title: Download -menu: - main: - weight: -2000 ---- +# Download -## Releases +## Install a release -See the [Releases](https://github.com/ohsu-comp-bio/funnel/releases) page for release history. +The [latest stable release](https://github.com/calypr/funnel/releases/latest) includes binaries for Linux and macOS on amd64 and arm64. Each release provides SHA-256 checksums. - ---8<-- "docs/tools/funnel/_releases.md" +Install the latest release: -Check out the [development docs][dev] for more detail. +```sh +curl -fsSL https://calypr.org/tools/funnel/install.sh | bash +``` -[dev]: ./development/developers.md +The default destination is `$HOME/.local/bin`. To install a specific version, pass its tag: + +```sh +curl -fsSL https://calypr.org/tools/funnel/install.sh | bash -s -- v0.12.3 +``` + +See [all releases and release notes](https://github.com/calypr/funnel/releases). +Before replacing an existing deployment, read the [upgrade guide](upgrading.md). + +## Container images + +Images for both amd64 and arm64 are published to [Quay](https://quay.io/repository/ohsu-comp-bio/funnel?tab=tags). +Use a version tag for a fixed release: + +```sh +docker pull quay.io/ohsu-comp-bio/funnel:v0.12.3 +``` + +Stable releases also update `latest`. Start a server with Docker: + +```sh +docker run --rm -p 8000:8000 quay.io/ohsu-comp-bio/funnel:latest server run +``` + +Configure persistent task storage before using this temporary container for an existing deployment. See [deployment configuration](compute/deployment.md). diff --git a/docs/tools/funnel/docs/events.md b/docs/tools/funnel/docs/events.md new file mode 100644 index 00000000..1f187df0 --- /dev/null +++ b/docs/tools/funnel/docs/events.md @@ -0,0 +1 @@ +# Events diff --git a/docs/tools/funnel/docs/events/file.md b/docs/tools/funnel/docs/events/file.md new file mode 100644 index 00000000..48892836 --- /dev/null +++ b/docs/tools/funnel/docs/events/file.md @@ -0,0 +1,31 @@ +# Event files + +Use the `file` event writer to save structured task events as newline-delimited JSON (NDJSON). Each +line follows Funnel's `events.Event` schema and can be decoded with +`events.Unmarshal`. Nested system-log fields remain nested. The `log` writer +continues to produce diagnostic output, which is not an event archive. + +```yaml +EventWriters: + - log + - file +EventFile: /var/log/funnel/events.ndjson +``` + +The parent directory must exist and be writable by Funnel. New files are created +with permissions `0600`. Existing files are appended to, with their permissions +unchanged. The file is closed when its owning event pipeline closes. Close failures are logged. The +`--EventFile` flag can also supply the path. + +Give each process its own output file. In particular, distributed workers must +use distinct paths or separate local filesystems. Writes are serialized within +one writer, but Funnel does not coordinate multiple processes writing the same +file. Files are not automatically rotated. + +File errors are returned through the existing event-writer error handling. +This output is not a durable message queue: writes are not individually synced +to disk, and a crash or failed write can leave an incomplete final line. Event +selection and log-level filtering in the configured pipeline still apply. + +Schema-compatible output does not by itself provide safe database replay. +Importing events requires handling ordering, duplicates, and partial failures. diff --git a/docs/tools/funnel/docs/events/kafka.md b/docs/tools/funnel/docs/events/kafka.md index 242ce934..3f4c98ce 100644 --- a/docs/tools/funnel/docs/events/kafka.md +++ b/docs/tools/funnel/docs/events/kafka.md @@ -1,10 +1,3 @@ ---- -title: Kafka -menu: - main: - parent: Events ---- - # Kafka Funnel supports writing task events to a Kafka topic. To use this, add an event diff --git a/docs/tools/funnel/docs/index.md b/docs/tools/funnel/docs/index.md deleted file mode 100644 index 58bf3df0..00000000 --- a/docs/tools/funnel/docs/index.md +++ /dev/null @@ -1,190 +0,0 @@ ---- -title: Funnel ---- - -## Funnel - -Funnel is designed to streamline data processing workflows, enabling efficient data transformation and analysis. Key features include: - -- **S3 Integration**: Seamlessly add and manage files from Amazon S3. -- **Data Transformation**: Predefined pipelines for common data processing tasks. -- **Automation**: Schedule and automate repetitive data workflows. -- **Monitoring**: Track the status and performance of data jobs in real-time. -- **Workflow engine compatibile**: Compatible with Nextflow - -## Simple API -A task describes metadata, state, input/output files, resource requests, commands, and logs. - -The task API has four actions: create, get, list, and cancel. - -##Standards based - -The Task API is developed via an open standard effort. The TES API was developed as part of the Global Alliance for Genomics and Health (GA4GH). - -## Cross platform -We use Funnel on AWS, Google Cloud, OpenStack, and the good ol' university HPC cluster. - -## Adaptable - -A wide variety of options make Funnel easily adaptable: - - - BoltDB - - Elasticsearch - - MongoDB - - AWS Batch, S3, DynamoDB - - OpenStack Swift - - Google Cloud Storage, Datastore - - Kafka - - HPC support: HTCondor, Slurm, etc. - - and more - - --- - -# Define a task - -A task describes metadata, state, input/output files, resource requests, commands, and logs. - -For a full description of the task fields, see the task API docs and the the task schema. - - -``` -$ funnel examples hello-world -{ - "name": "Hello world", - "description": "Demonstrates the most basic echo task.", - "executors": [ - { - "image": "alpine", - "command": ["echo", "hello world"], - } - ] -} -``` - ---- - -# Start a Funnel server - -localhost:8000 is the HTTP API and web dashboard. -localhost:9090 is the gRPC API (for internal communication) - - -``` -$ funnel server run -server Server listening -httpPort 8000 -rpcAddress :9090 -``` - - ---- - -# Create a task - -The output is the task ID. - -This example uses the development server, which will run the task locally via Docker. - - -``` -$ funnel examples hello-world > hello-world.json -$ funnel task create hello-world.json -b8581farl6qjjnvdhqn0 -``` - ---- - -# Get the task - -The output is the task with state and logs. - -By default, the CLI returns the "full" task view, which includes all logs plus stdout/err content. - - -``` -$ funnel task get b8581farl6qjjnvdhqn0 -{ - "id": "b8581farl6qjjnvdhqn0", - "state": "COMPLETE", - "name": "Hello world", - "description": "Demonstrates the most basic echo task.", - "executors": [ - { - "image": "alpine", - "command": [ - "echo", - "hello world" - ], - } - ], - "logs": [ - { - "logs": [ - { - "startTime": "2017-11-13T21:35:57.548592769-08:00", - "endTime": "2017-11-13T21:36:01.871905687-08:00", - "stdout": "hello world\n" - } - ], - "startTime": "2017-11-13T21:35:57.547408797-08:00", - "endTime": "2017-11-13T21:36:01.87496482-08:00" - } - ], - "creationTime": "2017-11-13T21:35:57.543528992-08:00" -} -``` - ---- - -# List the tasks - - -``` -$ funnel task list --view MINIMAL -{ - "tasks": [ - { - "id": "b8581farl6qjjnvdhqn0", - "state": "COMPLETE" - }, - ... - ] -} -``` - ---- - -# Quickly create tasks - -The "run" command makes it easy to quickly create a task. By default, commands are wrapped in "sh -c" and run in the "alpine" container. - -Use the "--print" flag to print the task instead of running it immediately. - - -``` -$ funnel run 'md5sum $src' --in src=~/src.txt --print -{ - "name": "sh -c 'md5sum $src'", - "inputs": [ - { - "name": "src", - "url": "file:///Users/buchanae/src.txt", - "path": "/inputs/Users/buchanae/src.txt" - } - ], - "executors": [ - { - "image": "alpine", - "command": [ - "sh", - "-c", - "md5sum $src" - ], - "env": { - "src": "/inputs/Users/buchanae/src.txt" - } - } - ], -} - -``` diff --git a/docs/tools/funnel/docs/integrations/nextflow.md b/docs/tools/funnel/docs/integrations/nextflow.md index 3090a941..d8e8c226 100644 --- a/docs/tools/funnel/docs/integrations/nextflow.md +++ b/docs/tools/funnel/docs/integrations/nextflow.md @@ -1,19 +1,12 @@ ---- -title: Nextflow -menu: - main: - parent: Integrations ---- +# Nextflow > ⚠️ Nextflow support is currently in development and requires a few additional steps to run which are included below. -# Nextflow - -[Nextflow](https://nextflow.io/) is a workflow engine with a [rich ecosystem]() of pipelines centered around biological analysis. +[Nextflow](https://nextflow.io/) is a workflow engine with a rich ecosystem of pipelines centered around biological analysis. > Nextflow enables scalable and reproducible scientific workflows using software containers. It allows the adaptation of pipelines written in the most common scripting languages. -> Its fluent DSL simplifies the implementation and the deployment of complex parallel and reactive workflows on clouds and clusters. +> Its fluent DSL simplifies the implementation and the deployment of complex parallel and reactive workflows on clouds and clusters. Since Nextflow [includes support](https://www.nextflow.io/docs/latest/executor.html#ga4gh-tes) for the TES API, it can be used in conjunction with Funnel to run tasks or to interact with a common TES endpoint. @@ -62,7 +55,7 @@ plugins { } process.executor = 'tes' -tes.endpoint = 'http://localhost:8000' # <--- Funnel's default address +tes.endpoint = 'http://localhost:8000' # <--- Funnel's default address EOF ``` @@ -73,7 +66,7 @@ Start the Funnel server: ```sh funnel server run ``` - + ### 4. Run Nextflow In another window, run the workflow: @@ -91,10 +84,10 @@ nextflow run main.nf -c nextflow.config - [Nextflow's TES Support](https://www.nextflow.io/docs/latest/executor.html#ga4gh-tes) - [nf-core](https://nf-co.re/) - > A community effort to collect a curated set of analysis pipelines built using Nextflow. + > A community effort to collect a curated set of analysis pipelines built using Nextflow. - [nf-canary](https://github.com/seqeralabs/nf-canary) - > A minimal Nextflow workflow for testing infrastructure. + > A minimal Nextflow workflow for testing infrastructure. - [Nextflow Patterns](https://nextflow-io.github.io/patterns/) - > A curated collection of Nextflow implementation patterns + > A curated collection of Nextflow implementation patterns diff --git a/docs/tools/funnel/docs/integrations/py-tes.md b/docs/tools/funnel/docs/integrations/py-tes.md index 7b120613..d7ecb5cb 100644 --- a/docs/tools/funnel/docs/integrations/py-tes.md +++ b/docs/tools/funnel/docs/integrations/py-tes.md @@ -1,14 +1,7 @@ ---- -title: py-tes -menu: - main: - parent: Integrations ---- +# py-tes > ⚠️ py-tes support is in active development and may be subject to change. -# py-tes - [py-tes](https://github.com/ohsu-comp-bio/py-tes) is a library for interacting with servers implementing the [GA4GH Task Execution Schema](https://github.com/ga4gh/task-execution-schemas). ## Getting Started diff --git a/docs/tools/funnel/docs/integrations/remote-server.md b/docs/tools/funnel/docs/integrations/remote-server.md new file mode 100644 index 00000000..f3c1f759 --- /dev/null +++ b/docs/tools/funnel/docs/integrations/remote-server.md @@ -0,0 +1,40 @@ +# Connect to a remote Funnel server + +By default, the Funnel CLI connects to `http://localhost:8000`. Use `-S` to +select another server for a command: + +```sh +funnel -S https://funnel.example.com task list +``` + +To use the same server for multiple commands, set `FUNNEL_SERVER`: + +```sh +export FUNNEL_SERVER=https://funnel.example.com +funnel task list +``` + +The `-S` flag takes precedence when both are set. + +Set `FUNNEL_SERVER_USER` and `FUNNEL_SERVER_PASSWORD` to authenticate using +Basic authentication. Credentials require HTTPS for remote HTTP servers. +Plain HTTP is allowed for loopback development (`localhost`, `127.0.0.1`, or +`[::1]`). Authenticated HTTPS requests cannot redirect to HTTP. + +For a server configured with OIDC authentication, set `FUNNEL_SERVER_TOKEN` +to a valid bearer token. The HTTP task client sends it in the `Authorization` +header instead of Basic credentials when both are configured. The same HTTPS +and redirect restrictions apply. With no bearer token, existing Basic +authentication behavior is preserved, including credentials with an empty +password. The CLI does not refresh expired tokens; replace the environment +value and rerun the command when needed. + +The internal gRPC client currently uses plaintext transport. It rejects +credentials for remote addresses by default; use a secure tunnel terminating +on loopback. This restriction does not add gRPC TLS support. + +For a known trusted private deployment that requires remote plaintext +credentials, explicitly set `FUNNEL_ALLOW_INSECURE_CREDENTIALS=true`. This +also permits HTTP credential downgrades, and exposes credentials to anyone +who can observe that connection. Unauthenticated connections remain available +without this setting. diff --git a/docs/tools/funnel/docs/metrics.md b/docs/tools/funnel/docs/metrics.md new file mode 100644 index 00000000..b05321b8 --- /dev/null +++ b/docs/tools/funnel/docs/metrics.md @@ -0,0 +1 @@ +# Metrics diff --git a/docs/tools/funnel/docs/metrics/prometheus.md b/docs/tools/funnel/docs/metrics/prometheus.md index c59894b4..08a04a11 100644 --- a/docs/tools/funnel/docs/metrics/prometheus.md +++ b/docs/tools/funnel/docs/metrics/prometheus.md @@ -1,10 +1,3 @@ ---- -title: Prometheus -menu: - main: - parent: Metrics ---- - # Prometheus [Prometheus][prom] is a monitoring and metrics collection service. It pulls metrics @@ -32,4 +25,5 @@ Funnel exports these metrics: of bytes of disk space available by all nodes. [prom]: https://prometheus.io/ +[gauge]: https://prometheus.io/docs/concepts/metric_types/#gauge [graf]: https://grafana.com/ diff --git a/docs/tools/funnel/docs/security.md b/docs/tools/funnel/docs/security.md new file mode 100644 index 00000000..8dbb2f9b --- /dev/null +++ b/docs/tools/funnel/docs/security.md @@ -0,0 +1 @@ +# Security diff --git a/docs/tools/funnel/docs/security/advanced.md b/docs/tools/funnel/docs/security/advanced.md index 8812d759..919f1e1f 100644 --- a/docs/tools/funnel/docs/security/advanced.md +++ b/docs/tools/funnel/docs/security/advanced.md @@ -1,11 +1,3 @@ ---- -title: Advanced Auth -menu: - main: - parent: Security - weight: 10 ---- - # Overview 🔐 Thanks to our collaborators at CTDS — Funnel is currently adding support for "Per-User/Per-Bucket" credentials to allow Users to access S3 Buckets without having to store their credentials in the Funnel Server. @@ -21,7 +13,8 @@ In this way this Wrapper can manage the bucket and the keys (the Wrapper would Stay tuned for this feature's development! This feature is being tracked with the following: -- Pull Request: https://github.com/calypr/funnel/pull/1098 +- GitHub Branch: https://github.com/ohsu-comp-bio/funnel/tree/feature/credentials +- Pull Request: https://github.com/ohsu-comp-bio/funnel/pull/1098 # Credits 🙌 diff --git a/docs/tools/funnel/docs/security/basic.md b/docs/tools/funnel/docs/security/basic.md index 0b19e071..cdd8b730 100644 --- a/docs/tools/funnel/docs/security/basic.md +++ b/docs/tools/funnel/docs/security/basic.md @@ -1,10 +1,3 @@ ---- -title: Basic Auth -menu: - main: - parent: Security - weight: 10 ---- # Basic Auth By default, a Funnel server allows open access to its API endpoints, but it @@ -40,10 +33,15 @@ so you will also need to configure the RPC client. ```yaml RPCClient: - User: funnel - Password: abc123 + Credential: + User: funnel + Password: abc123 ``` +Beginning with Funnel v0.11.8, `User` and `Password` must be nested under +`RPCClient.Credential`. Configurations created for earlier releases need to be +updated to use this structure. + Make sure to properly protect the configuration file so that it's not readable by everyone: @@ -57,3 +55,13 @@ $ export FUNNEL_SERVER_USER=funnel $ export FUNNEL_SERVER_PASSWORD=abc123 $ funnel task list ``` + +## API audit logs + +The server writes structured `AUDIT` entries at Info level for unary gRPC API +requests, including calls through the HTTP gateway and authentication failures. +Entries record the method, user ID, result code, and task ID when available. +Public or unauthenticated requests use `ANONYMOUS`; internal calls without a +user context use `SYSTEM`. Request bodies, bearer tokens, and raw error or panic +values are excluded. The configured logger controls output and log level; +these entries use the same logging destination as other server messages. diff --git a/docs/tools/funnel/docs/security/oauth2.md b/docs/tools/funnel/docs/security/oauth2.md index 4b4232da..0738acc1 100644 --- a/docs/tools/funnel/docs/security/oauth2.md +++ b/docs/tools/funnel/docs/security/oauth2.md @@ -1,10 +1,3 @@ ---- -title: OAuth2 -menu: - main: - parent: Security - weight: 10 ---- # OAuth2 By default, a Funnel server allows open access to its API endpoints, but in @@ -18,8 +11,9 @@ In addition, if the OIDC provides a token introspection endpoint (in its configuration JSON), Funnel server also calls that endpoint to make sure the token is still active (i.e., no token invalidation before expiring). -Optionally, Funnel can also validate the scope and audience claims to contain -specific values. +Funnel requires an expiration claim and an explicitly configured API audience. +It can also require a specific scope. Set the audience expected by your API; +this may differ from the OAuth client ID. To enable JWT authentication, specify `OidcAuth` section in your config file: @@ -37,7 +31,7 @@ Server: # Optional: if specified, this scope value must be in the token: RequireScope: funnel-id - # Optional: if specified, this audience value must be in the token: + # Required: this API audience value must be in the token: RequireAudience: tes-api # The URL where OIDC should redirect after login (keep the path '/login') diff --git a/docs/tools/funnel/docs/storage.md b/docs/tools/funnel/docs/storage.md new file mode 100644 index 00000000..e0e8ac41 --- /dev/null +++ b/docs/tools/funnel/docs/storage.md @@ -0,0 +1 @@ +# Storage diff --git a/docs/tools/funnel/docs/storage/ftp.md b/docs/tools/funnel/docs/storage/ftp.md index 79b2439d..f14a57c7 100644 --- a/docs/tools/funnel/docs/storage/ftp.md +++ b/docs/tools/funnel/docs/storage/ftp.md @@ -1,17 +1,29 @@ ---- -title: FTP -menu: - main: - parent: Storage ---- - # FTP Funnel supports download and uploading files via FTP. -Currently authentication credentials are take from the URL, e.g. `ftp://username:password@ftp.host.tld`. This will be improved soon to allow credentials to be added to the configuration file. +Credentials may come from the URL, for example +`ftp://username:password@ftp.host.tld/file`, or from the worker configuration. +Configured credentials require a trusted `Endpoint`: + +```yaml +FTPStorage: + Endpoint: "ftp://ftp.host.tld:21" + User: "username" + Password: "password" +``` + +Funnel sends configured credentials only when the requested hostname and port +match that endpoint. Hostname comparison ignores case; an omitted port means +`21`. A different origin is rejected before connecting. Existing deployments +using configured credentials must set `Endpoint` when upgrading. The endpoint +must be an FTP origin without userinfo, a file path, a query, or a fragment. + +Credentials explicitly included in a task URL replace the configured credentials +completely, including an omitted URL password. The public anonymous defaults +(`anonymous` / `anonymous`) do not require an endpoint. -The FTP storage client is enabled by default, but may be explicitly disabled in the +The FTP storage client is enabled by default, but may be explicitly disabled in the worker config: ```yaml diff --git a/docs/tools/funnel/docs/storage/google-storage.md b/docs/tools/funnel/docs/storage/google-storage.md index d8fde4f3..50f9a8a0 100644 --- a/docs/tools/funnel/docs/storage/google-storage.md +++ b/docs/tools/funnel/docs/storage/google-storage.md @@ -1,10 +1,3 @@ ---- -title: Google Storage -menu: - main: - parent: Storage ---- - # Google Storage Funnel supports using [Google Storage][gs] (GS) for file storage. diff --git a/docs/tools/funnel/docs/storage/http.md b/docs/tools/funnel/docs/storage/http.md index 81922053..da32fe19 100644 --- a/docs/tools/funnel/docs/storage/http.md +++ b/docs/tools/funnel/docs/storage/http.md @@ -1,26 +1,56 @@ ---- -title: HTTP(S) -menu: - main: - parent: Storage ---- - # HTTP(S) -Funnel supports downloading files from public URLs via GET requests. No authentication -mechanism is allowed. This backend can be used to fetch objects from cloud storage -providers exposed using presigned URLs. +Funnel can download inputs from and upload outputs to HTTP(S) URLs that need no +credentials: public URLs, or presigned URLs issued by a cloud storage provider +such as Amazon S3 or Google Cloud Storage. No authentication mechanism is +allowed, so the URL itself must grant access. + +- Inputs are downloaded with a GET request. +- Outputs are uploaded with a PUT request that carries a `Content-Length` + header, as presigned S3 upload URLs require. + +Presigned URLs work for `FILE` inputs and outputs. A presigned URL is signed for +one exact object, but Funnel derives the URL of each file in a `DIRECTORY` +output, and of each match of a wildcard output, by appending the file's path to +the path of the output URL, keeping its query string. Those derived URLs carry +no valid signature, so use one `FILE` output per presigned URL. + +For `DIRECTORY` and wildcard outputs, point the output URL at a server that +signs each upload as it arrives, when the file name is known, and redirects it +to the storage provider: -The HTTP storage client is enabled by default, but may be explicitly disabled in the +- An upload follows a 307 or 308 redirect and resends the file to the new URL. +- Uploads send `Expect: 100-continue` and wait up to one second for the server + to answer before sending the file. A server that redirects within that second + doesn't receive the file; a slower one may receive some or all of it first. +- A 301, 302 or 303 response to an upload is reported as an error, because + following it would turn the upload into a GET. +- Downloads follow redirects too. + +The URL is stored with the task, including any signature in its query string, +so issue presigned URLs with a short expiry. The storage provider limits the +size of a single PUT (5 GB on Amazon S3). + +The HTTP storage client is enabled by default, but may be explicitly disabled in the worker config: ```yaml HTTPStorage: Disabled: false - # Timeout for http(s) GET requests. - Timeout: 30s + # Optional total request deadline, including the entire file transfer. + Timeout: + disabled: true ``` +By default, HTTP transfers have no overall deadline. Go's default transport +limits connection and TLS handshake waits. Funnel waits up to one minute for +response headers after sending the request. For downloads, this starts after +sending the GET request. For uploads, it starts after sending the file. Task +cancellation stops transfers. Set `Timeout.duration` to a duration such as `30s` +to limit the whole request. There is no separate transfer inactivity timeout, so +a response body that stops sending data can wait until canceled or until the +configured total deadline expires. + ### Example task ```json { @@ -29,9 +59,14 @@ HTTPStorage: "url": "http://fakedomain.com/hello.txt", "path": "/inputs/hello.txt" }], + "outputs": [{ + "url": "https://my-bucket.s3.amazonaws.com/hello-upper.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Signature=...", + "path": "/outputs/hello-upper.txt", + "type": "FILE" + }], "executors": [{ "image": "alpine", - "command": ["cat", "/inputs/hello.txt"], + "command": ["sh", "-c", "tr a-z A-Z < /inputs/hello.txt > /outputs/hello-upper.txt"] }] } ``` diff --git a/docs/tools/funnel/docs/storage/index.md b/docs/tools/funnel/docs/storage/index.md deleted file mode 100644 index 8e2b7abf..00000000 --- a/docs/tools/funnel/docs/storage/index.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -title: Storage ---- - -# Storage - -Funnel storage backends handle downloading task inputs and uploading task outputs. -Each input/output URL in a task is resolved by its URL scheme (for example `s3://`, -`gs://`, or `file://`). - -### Choosing a storage backend - -- Use [Local](/tools/funnel/docs/storage/local/) for files on worker-accessible disks. -- Use [HTTP(S)](/tools/funnel/docs/storage/http/) for public URLs and presigned object links. -- Use [FTP](/tools/funnel/docs/storage/ftp/) for FTP-hosted files. -- Use [S3](/tools/funnel/docs/storage/s3/) for Amazon S3 and S3-compatible object stores. -- Use [OpenStack Swift](/tools/funnel/docs/storage/swift/) for Swift object storage. -- Use [Google Storage](/tools/funnel/docs/storage/google-storage/) for Google Cloud Storage. - -### URL schemes - -| Backend | URL scheme | -| --- | --- | -| Local | `file://` | -| HTTP(S) | `http://`, `https://` | -| FTP | `ftp://` | -| S3 | `s3://` | -| OpenStack Swift | `swift://` | -| Google Storage | `gs://` | - -### Notes - -- Most cloud/object backends can load credentials from the environment, but each backend page includes explicit config options. -- Storage clients can be enabled/disabled in worker configuration. -- The worker process performs all transfer operations before and after executor commands run. diff --git a/docs/tools/funnel/docs/storage/local.md b/docs/tools/funnel/docs/storage/local.md index eb686699..d87ec135 100644 --- a/docs/tools/funnel/docs/storage/local.md +++ b/docs/tools/funnel/docs/storage/local.md @@ -1,16 +1,8 @@ ---- -title: Local -menu: - main: - parent: Storage - weight: -10 ---- - # Local Funnel supports using the local filesystem for file storage. -Funnel limits which directories may be accessed, by default only allowing directories +Funnel limits which directories may be accessed, by default only allowing directories under the current working directory of the Funnel worker. Config: @@ -46,18 +38,21 @@ Files must be absolute paths in `file:///path/to/file.txt` URL form. } ``` -### File hard linking behavior +### Directory boundaries and file transfers + +Funnel checks storage URLs against `AllowedDirs` during reads, writes, and listings. +Paths and relative symlinks cannot escape the selected allowed directory. Absolute +symlinks are rejected, including those pointing inside an allowed directory. Use +relative symlinks for files within that directory. -For efficiency, Funnel will attempt not to copy the input files, instead trying -create a hard link to the source file. In some cases this isn't possible. For example, -if the source file is on a network file system mount (e.g. NFS) but the Funnel worker's -working directory is on the local scratch disk, a hard link would cross a file system -boundary, which is not possible. In this case, Funnel will copy the file. +Local transfers copy files rather than creating hard links. This keeps file access +within the allowed directory during the transfer, but requires space for the copy. +The worker's task staging paths are separate from the storage URL allowlist. ### File ownership behavior -One difficult area of files and Docker containers is file owner/group management. -If a Docker container runs as root, it's likely that the file will end up being owned -by root on the host system. In this case, some step (Funnel or another task) will -likely fail to access it. This is a tricky problem with no good solution yet. +One difficult area of files and Docker containers is file owner/group management. +If a Docker container runs as root, it's likely that the file will end up being owned +by root on the host system. In this case, some step (Funnel or another task) will +likely fail to access it. This is a tricky problem with no good solution yet. See [issue 66](https://github.com/ohsu-comp-bio/funnel/issues/66). diff --git a/docs/tools/funnel/docs/storage/mounted-inputs.md b/docs/tools/funnel/docs/storage/mounted-inputs.md new file mode 100644 index 00000000..2adeb067 --- /dev/null +++ b/docs/tools/funnel/docs/storage/mounted-inputs.md @@ -0,0 +1,111 @@ +# Mounted inputs + +`GenericS3.MountedRead` and `GenericS3.MountedWrite` let a Kubernetes worker read inputs or publish outputs through PVC-backed filesystem mounts. Each setting names a configured storage volume. If that volume has multiple worker mounts, use `SubPath` to select the mount whose static Kubernetes `subPath` matches exactly. Read and write routes are optional and can use separate volumes or mount views. + +> **Warning:** `MountedWrite` checks only whether an S3 object exists with the expected size. If an older object at the output URL has the same size, Funnel can report the task as complete before the new output is visible in S3. Use the regular S3 API upload path when output URLs can be reused or when you need confirmation that the new content was published. Omit `MountedWrite` to use API uploads. + +> Mounted reads also depend on the storage driver presenting a fresh view of the configured bucket and prefix. Funnel cannot verify that the mount matches the S3 URL. + +```yaml +GenericS3: + - Disabled: false + Endpoint: https://s3.example.org + Bucket: analysis-data + Key: storage-key + Secret: storage-secret + Region: us-east-1 + MountedRead: + Volume: reference-data + SubPath: genomes + Prefix: genomes/ + MountedWrite: + Volume: result-data + Prefix: results/ + Timeout: 2700s + +Kubernetes: + JobsNamespace: funnel-tasks + Storage: + WorkspaceVolume: workspace + volumes: + - name: workspace + persistentVolumeClaim: + claimName: task-workspace + - name: reference-data + persistentVolumeClaim: + claimName: reference-pvc + - name: result-data + persistentVolumeClaim: + claimName: task-results + volumeMounts: + - name: reference-data + mountPath: /mnt/reference + readOnly: true + subPath: genomes + - name: reference-data + mountPath: /mnt/annotations + readOnly: true + subPath: annotations + - name: result-data + mountPath: /mnt/results +``` + +The two reference mounts use one PVC at different paths. `MountedRead.SubPath: genomes` selects the first view. `MountedWrite` omits `SubPath` because `result-data` has one worker mount. If a volume has several mounts, set `SubPath` to the exact static `volumeMounts[].subPath` value. Set `SubPath: ""` to select a static root mount. A mount with `subPathExpr` can be selected only when it is the unique mount for that volume and `SubPath` is absent. + +`Prefix` is relative to the bucket. An empty prefix selects the whole bucket. The selected mount's authored `mountPath` is the route root. For example, `s3://analysis-data/genomes/genome.fa` maps to `/mnt/reference/genome.fa`. Funnel does not append the selected subpath or prefix to `mountPath`. The storage driver and operator must make the mounted view correspond to the configured bucket prefix and provide the required freshness. + +`MountedWrite` uses the same volume, subpath, and prefix rules. Its selected mount must be writable, and the underlying PVC must permit writes. Before creating task resources, Funnel checks both existing claims and PVC templates for `ReadWriteOnce`, `ReadWriteMany`, or `ReadWriteOncePod` access. A claim with only `ReadOnlyMany` cannot serve mounted outputs. The workspace still cannot use `ReadWriteOncePod` because worker and executor Pods share it. Funnel appends an output key path relative to the configured prefix when it copies a TES output file into the mount. + +For native storage configuration details, including existing claims, StorageClass-provisioned claims, and explicit PV/PVC pairs, see [Kubernetes storage](../compute/kubernetes-storage.md). + +## TES behavior + +Clients continue to submit ordinary TES inputs. They do not supply PVC names, volume names, worker mount paths, or deployment paths. + +```json +{ + "inputs": [ + { + "url": "s3://analysis-data/genomes/genome.fa", + "path": "/inputs/genome.fa", + "type": "FILE" + } + ], + "executors": [ + { + "image": "alpine:3.21", + "command": ["cat", "/inputs/genome.fa"] + } + ] +} +``` + +The worker copies each input into the task workspace. For a matching GenericS3 mounted read, it reads the source from the selected filesystem mount. The executor sees the copy at `/inputs/genome.fa`. Changing that copy does not change the mounted source. Directory inputs preserve their relative filenames. Reference mounts remain worker-only. They do not change TES `volumes`, which retain their empty scratch-directory behavior. + +## Read routing and safety + +The worker activates a mounted read or write only when the submitted storage plan names the exact GenericS3 `Endpoint` and `Bucket`. The plan stores `S3ReadMounts` and `S3WriteMounts` with the endpoint, bucket, prefix, and mount root, without credentials. The worker ignores static `MountedRead` and `MountedWrite` settings, including when the submitted plan has no mounted routes. Configured mounted reads and writes require `Kubernetes.Storage` and its submitted plan. The API server's `NewMux` does not activate filesystem reads or writes. + +GenericS3 handles inputs whose bucket-relative key matches its configured prefix. Funnel treats S3 keys literally. Spaces, `#`, and percent signs remain part of the key. For example, `a%20b.txt` names the literal file `a%20b.txt`. Prefix matching uses whole path components, so `genomes/` does not match `genomes-old/file.fa`. Unmatched URLs use the configured storage API path. + +A matched file must exist beneath the selected mount root. Funnel rejects traversal, encoded traversal forms, and escaping symlinks through rooted filesystem operations. Regular-file symlinks within the root can be read. Directory listings reject directory symlinks and special files. A missing or unsafe matched input fails the transfer. Funnel does not fall back to the API for that input. + +## Configuration ownership + +The server resolves each backend's selected worker volume mount and records mounted reads and writes in version 4 storage plans. Workers use the plan directly, even when their static ConfigMap differs. The server does not read worker filesystem paths. Existing Jobs with version 4 plans remain compatible. The version 4 decoder cannot run or clean up Jobs with version 1, 2, or 3 plans. Finish those tasks and clean up their Jobs with a compatible older binary before upgrading. + +The broader native storage migration changes the config schema. Replace `MountedRead.WorkerMount` and `MountedWrite.WorkerMount` with `Volume`, and set `SubPath` when one volume has multiple explicit worker mounts. Rebuild Go RPC plugins against the updated config package before exchanging configuration with them. The automatic PVC naming change does not alter the plugin schema or require another rebuild. The schema reserves field 1 and the name `WorkerMount` in `MountedRead` and `MountedWrite`. Other compute backends retain their existing storage clients. No TES schema fields change. + +## Output publication + +For an output URL that matches a configured `MountedWrite` endpoint, bucket, and prefix, the worker copies each output file to a private temporary file beside its destination. After closing the file, it renames the temporary file over the destination. Concurrent transfers publish complete files instead of copying into the same inode. If the source already is the mounted destination, Funnel skips the copy. It then calls S3 `HeadObject` until the object exists with the same size as the local file. The worker checks zero-byte files too. + +Mounted file transfers require Unix. The filesystem must support exclusive temporary-file creation, POSIX file permissions, and atomic replacement through same-directory rename, including across worker Pods. A failed rename fails the transfer. Funnel does not retry through an in-place copy or an API upload. Separate write routes must use nonoverlapping filesystem views. Funnel rejects overlapping roots and overlapping views of the same PVC that it can identify from the storage plan. + +After a failed or canceled copy, Funnel attempts to remove its temporary file and reports cleanup errors. A process crash can leave temporary files, and a storage driver can export temporary names to the bucket before Funnel removes them. Account for these files in the deployment's storage cleanup policy. Filesystem replacement does not make the driver's later S3 synchronization atomic. See [S3 Files synchronization](https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-files-synchronization.html) for that driver's rename and export behavior. + +This check proves that S3 reports an object with the expected size. It does not compare file contents, checksums, or object versions. A same-sized object left by an earlier task can satisfy the check. Set `MountedWrite.Timeout` to bound polling. The example's `2700s` timeout is 45 minutes. The default is three hours, and the worker polls every two seconds. An explicit timeout must be positive. Task cancellation also stops the wait. + +The GenericS3 credentials must allow `HeadObject` for mounted destinations. They do not need upload permission for an output handled by the mount. An output URL that does not match the configured endpoint, bucket, and prefix follows the regular GenericS3 API upload path, which requires upload permission. Outputs do not use a `MountedRead` setting. + +Funnel handles directory outputs as individual files. It records successful files in TES output logs and preserves partial successes when another file fails. A timeout, cancellation, or transfer error prevents successful task completion. Retries retain one successful log entry per file after publication succeeds. diff --git a/docs/tools/funnel/docs/storage/s3.md b/docs/tools/funnel/docs/storage/s3.md index 75a02711..d9089c3f 100644 --- a/docs/tools/funnel/docs/storage/s3.md +++ b/docs/tools/funnel/docs/storage/s3.md @@ -1,10 +1,3 @@ ---- -title: S3 -menu: - main: - parent: Storage ---- - # S3 ## Amazon S3 @@ -18,16 +11,27 @@ may explicitly set the credentials in the worker config: ```yaml AmazonS3: Disabled: false - # The maximum number of times that a request will be retried for failures. - MaxRetries: 10 - Key: "" - Secret: "" + AWSConfig: + # The maximum number of times that a request will be retried for failures. + MaxRetries: 10 + Key: "" + Secret: "" ``` -The Amazon S3 storage client also supports SSE-KMS and SSE-C configurations. +The Amazon S3 storage client supports SSE-S3, SSE-KMS, and SSE-C. + +For a bucket policy that requires the `AES256` encryption header, enable SSE-S3: -For SSE-KMS as long as your credentials can access the KMS key used for the -given bucket, no special configuration is required. However, you can specifiy a +```yaml +AmazonS3: + SSE: + S3Managed: true +``` + +Do not combine `S3Managed` with `KMSKey` or `CustomerKeyFile`. + +For SSE-KMS as long as your credentials can access the KMS key used for the +given bucket, no special configuration is required. However, you can specifiy a specific KMS key if desired: ```yaml @@ -52,14 +56,14 @@ AmazonS3: CustomerKeyFile: "./sse-c.key" ``` -Note that this file will need to be available to all Funnel workers. +Note that this file will need to be available to all Funnel workers. ## Other S3 API Providers -Funnel also supports using non-Amazon S3 API providers ([Ceph][ceph], +Funnel also supports using non-Amazon S3 API providers ([Ceph][ceph], [Cleversafe][cleversafe], [Minio][minio], etc.) for file storage. -These other S3 storage clients are NOT enabled by default. You must configure them. +These other S3 storage clients are NOT enabled by default. You must configure them. This storage client also supports the [version 4 signing process](https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html). @@ -71,6 +75,17 @@ GenericS3: Secret: "" ``` +### Downloads into Mountpoint + +When GenericS3 downloads an input into a Mountpoint filesystem, it stages the +complete input in the worker's temporary directory before copying it to the +mount. Set `TMPDIR` to a non-FUSE filesystem that supports `0600` file permissions +and has enough space for concurrent input downloads. + +Source-read failures leave an existing destination intact. Errors after the +copy into Mountpoint begins can leave an incomplete destination and fail the +task. Replacing existing files requires Mountpoint's `--allow-overwrite` option. + ### Example task ```json { diff --git a/docs/tools/funnel/docs/storage/swift.md b/docs/tools/funnel/docs/storage/swift.md index 3de323ae..208abc9e 100644 --- a/docs/tools/funnel/docs/storage/swift.md +++ b/docs/tools/funnel/docs/storage/swift.md @@ -1,10 +1,3 @@ ---- -title: OpenStack Swift -menu: - main: - parent: Storage ---- - # OpenStack Swift Funnel supports using [OpenStack Swift][swift] for file storage. diff --git a/docs/tools/funnel/docs/tasks.md b/docs/tools/funnel/docs/tasks.md index ed9d25fd..9526b898 100644 --- a/docs/tools/funnel/docs/tasks.md +++ b/docs/tools/funnel/docs/tasks.md @@ -1,11 +1,3 @@ ---- -title: Tasks -menu: - main: - identifier: tasks - weight: -70 ---- - # Tasks A task defines a unit of work: @@ -45,7 +37,7 @@ The example task below downloads a file named `hello.txt` from S3 and calls `cat } ``` -Tasks have multiple "executors"; containers and commands run in a sequence. +Tasks have multiple "executors"; containers and commands run in a sequence. Funnel runs executors via Docker. Tasks also have state and logs: @@ -96,6 +88,89 @@ Tasks also have state and logs: There are logs for each task attempt and each executor. Notice that the stdout is conveniently captured by `logs[0].logs[0].stdout`. +## Output types and patterns + +An output can declare `"type": "FILE"` or `"type": "DIRECTORY"`. The Funnel worker +fails output collection if the declared type does not match the produced path. If +`type` is omitted, Funnel keeps its existing behavior and inspects the path to +determine whether to upload a file or the files within a directory. + +An output pattern containing `*` that matches no paths produces a warning and +uploads nothing for that pattern. Other outputs are still collected. This includes +`abc/*` when `abc` is a file. An empty directory output also produces a warning and +uploads no files. If a task requires a nonempty result, its executor must check +for that result and exit with an error when it is missing. + +Missing exact output paths, malformed patterns, explicit type mismatches, and +upload failures remain collection errors. If the executor succeeded, a collection +error puts the task in `SYSTEM_ERROR`. These checks apply to backends that run the +Funnel worker. GCP Batch uses a separate output path. + +## Container paths and output mounts + +Funnel's Docker and Kubernetes workers use filesystem mounts to make task files +available to executor containers. A file output normally mounts its parent +directory. A directory output mounts the directory itself. Choose a dedicated +task directory, such as `/task` or `/outputs`, that does not contain files the +container image needs or overlap mounts in your executor template. + +### Existing image contents + +Directory mounts hide any files already present at the destination in the +container image. For example, an input directory mounted at `/etc` hides the +image's existing `/etc` contents for that executor. The image's files are not +deleted, but they are inaccessible beneath the mount. Input files are not merged +with the image's directory contents. This follows ordinary filesystem mount +behavior; see [Docker's bind-mount documentation](https://docs.docker.com/engine/storage/bind-mounts/#bind-mounting-over-existing-data). + +Funnel does not support automatic merging of input directories with image +directories. Merging would require policies for conflicting files, permissions, +and symbolic links. Use a dedicated input path such as `/task/inputs`, and point +the command at that path. When replacing one configuration file, declare that +specific file as the input rather than mounting its entire parent directory. + +### Root mounts + +Mounting over the container root `/` is unsupported. This includes a volume at +`/`, a directory output at `/`, and ordinary file outputs such as `/result.json` +whose parent is `/`. Root-level input files can also require a root mount when +Funnel groups their input directories. Funnel rejects unsupported mappings before +starting the executor. Move these paths beneath a dedicated directory and update +the task's commands and path declarations together. Funnel does not rewrite paths +inside commands or scripts. + +For example, capture stdout at `/task/stdout` and publish that same path: + +```json +{ + "executors": [{ + "image": "alpine", + "command": ["echo", "hello"], + "stdout": "/task/stdout" + }], + "outputs": [{ + "path": "/task/stdout", + "url": "s3://results/stdout.txt", + "type": "FILE" + }] +} +``` + +Docker has a limited exception for root-level file outputs that exactly match an +executor's `stdout` or `stderr` path. Funnel mounts those individual files so later +executors can read them. These paths are taken from the task, not from fixed names +such as `/stdout`. A container can read and write these files, but cannot unlink +or replace their file mount points. Input and directory conflicts are rejected. + +Kubernetes does not use this exception. Its shared storage can require the +executor to create stream files itself, so pre-creating a file for a Kubernetes +file mount is not safe for every supported storage driver. Use stream paths under +a dedicated directory for tasks that must run on both backends. + +These are Funnel storage limitations, not TES requirements. TES does not require +parent-directory mounts. Rejecting `/` mounts also does not prevent a non-root +mount from hiding files already present in the container image. + ### Task API The API lets you create, get, list, and cancel tasks. @@ -261,7 +336,7 @@ GET /v1/tasks?page_token=1h123h12j2h3k } ``` -### Cancel +### Cancel Tasks cannot be modified by the user after creation, with one exception – they can be canceled. ``` @@ -269,6 +344,19 @@ POST /v1/tasks/b85l8tirl6qkqbhg8vj0:cancel ``` +### Docker resource limits + +By default, Funnel does not enforce task CPU and RAM requests on Docker +containers. To apply them as limits, set `Worker.Container.EnforceResources` +to `true`. Each specified CPU or RAM request is applied independently. A RAM +limit also disables container swap. A process that exceeds its RAM limit may +be killed by the kernel. Existing custom `Worker.Container.RunCommand` +templates must include the resource flags themselves. + +### GPU requests + +GPU tasks use backend-specific TES `resources.backend_parameters`. See [GPU tasks](compute/gpus.md) for supported settings, hardware selection, and deployment requirements. + ### Full task spec Here's a more detailed description of a task. @@ -492,3 +580,42 @@ For a full, in-depth spec, read the TES standard's [task_execution.proto](https: ``` [alpine]: https://hub.docker.com/_/alpine/ + +## Task response sizes + +Task lists honor the requested TES view. `MINIMAL` returns task IDs and states. +`BASIC` omits input content and executor stdout/stderr, but includes output-file +logs. `FULL` includes the full task. The dashboard requests `BASIC` for its list +and `FULL` for task details. + +List pages are limited by both the requested task count and a protobuf response +byte budget. A page can contain fewer tasks than requested; follow its +`next_page_token` until it is empty. Tasks are never truncated to fit a page. + +Treat continuation tokens as opaque. MongoDB and PostgreSQL tokens encode the +last returned creation time and ID, matching their deterministic sort order. +Legacy ID-only tokens remain accepted while their referenced task exists; new +tokens can resume even after that task is deleted. + +The server's gRPC response limit defaults to 4 MiB. List pages account for task records and the continuation token within that +ceiling. If one task cannot fit, Funnel +returns `RESOURCE_EXHAUSTED` with its task ID and suggests requesting `MINIMAL` +or increasing the configured limit. Smaller pages cannot make a single large +task smaller. + +To support larger task responses, configure the server before starting it: + +```yaml +Server: + MaxResponseBytes: 8388608 +``` + +`0` selects the default. The setting +also controls the receive limit of Funnel's internal HTTP-to-gRPC gateway, so +HTTP clients such as the dashboard and CLI can receive the supported response. +External gRPC clients must configure their own receive limit when requesting +larger responses. `GetTask` is subject to the same response ceiling. + +This is a protobuf transport budget, not a JSON-size or process-memory limit. +JSON encoding can be larger, and the server must still load each candidate task +before measuring it. Set deployment limits according to available resources. diff --git a/docs/tools/funnel/docs/upgrading.md b/docs/tools/funnel/docs/upgrading.md new file mode 100644 index 00000000..f93e01b6 --- /dev/null +++ b/docs/tools/funnel/docs/upgrading.md @@ -0,0 +1,256 @@ +# Upgrade Funnel + +Use this guide when upgrading to the unreleased Go package reorganization and +the accompanying configuration and scheduler changes. It covers downstream Go +applications, authorization plugins, and existing deployments. + +The Go package moves preserve TES message field numbers, JSON names, and service +identities. They do not preserve Go import paths. Configuration schema changes +have separate migration requirements described below. + +## Prepare the deployment + +1. Back up the task database, configuration, custom templates, and HPC submission + records. Test the upgrade and rollback against a restored copy first. +2. Finish tasks whose Kubernetes storage plans use versions 1, 2, or 3. Use a + compatible older binary to clean up their Jobs before upgrading. +3. For HPC jobs submitted without trusted submission records, cancel them using + the scheduler's own command before upgrading. +4. Stop older Funnel servers, schedulers, and nodes before starting upgraded + instances. Upgrade the participating servers, nodes, workers, and plugins + together. Do not mix scheduler versions during the assignment migration. + +See [Kubernetes storage ownership and upgrades](compute/kubernetes-storage.md#ownership-and-upgrades) +and [scheduler assignment storage upgrades](compute/deployment.md#assignment-storage-upgrade) +for the backend-specific requirements. + +## Update Go imports + +Use `github.com/calypr/funnel` as the module prefix. Replace the former +`github.com/ohsu-comp-bio/funnel` prefix if your application still uses it. +The paths in this table are relative to that prefix. + +| Previous import or use | Replacement | +| --- | --- | +| `tes` request, response, and enum types | `api/tes` | +| `tes.NewClient` and `tes.Client` | `client/tes.NewClient` and `client/tes.Client` | +| `config` schema types | `api/config` | +| `logger` configuration schema types | `api/logger` | +| `plugins/proto` messages and `plugins/shared` interfaces, handshake, and adapters | `api/plugin` | +| `server`, `compute`, `worker`, `storage`, `database`, `events`, and other application implementations | Private under `internal/`. Use the REST client or plugin contracts for external integrations. | + +There are no forwarding packages for the old imports. Go rejects external +imports of `internal/` packages. The generated Go gRPC and gateway bindings are +also internal. Use the REST client, or generate bindings in your own module if +you maintain a direct gRPC integration. + +Replace `tes.Marshaler`, `tes.MarshalToString`, and the client's `Marshaler` +field with explicit `google.golang.org/protobuf/encoding/protojson` calls. +Application defaults, YAML loading, validation, redaction, asset access, runtime +logging, and plugin process management are no longer public library functions. +For plugins, use the configuration supplied by the host. For external tooling, +own the required configuration loading and validation instead of importing the +application implementation. `funnel examples default-config` prints the bundled +configuration example. + +The public `api/config` and `api/logger` packages contain schema types. They do +not replace the old runtime helpers such as `config.DefaultConfig`, `ParseFile`, +`ToYamlFile`, or `logger.NewLogger`. The former `util` helpers have moved to their +owning packages or the standard library. Build metadata is public in `version`. + +### Configure the REST client + +Import the client separately from the TES models. Credentials can come from the +environment or be set on the client: + +```go +package example + +import ( + "context" + "net/http" + "time" + + "github.com/calypr/funnel/api/tes" + tesclient "github.com/calypr/funnel/client/tes" +) + +func GetTask(ctx context.Context, address, token, id string) (*tes.Task, error) { + client, err := tesclient.NewClient(address, + tesclient.WithHTTPClient(&http.Client{Timeout: 30 * time.Second})) + if err != nil { + return nil, err + } + client.Token = token + return client.GetTask(ctx, &tes.GetTaskRequest{Id: id, View: "FULL"}) +} +``` + +`NewClient` continues to read `FUNNEL_SERVER_USER`, `FUNNEL_SERVER_PASSWORD`, +`FUNNEL_SERVER_TOKEN`, and `FUNNEL_ALLOW_INSECURE_CREDENTIALS` at construction. +These variables remain supported by both Go applications and the CLI. Set +`User`, `Password`, or `Token` afterward, or supply a `ClientOption`, to override +the environment defaults. A token takes precedence over Basic credentials. + +Use HTTPS for credentials sent to remote hosts. The REST client allows loopback +HTTP. For deployments that deliberately permit remote plaintext credentials, +pass `tesclient.WithInsecureCredentials()` in Go or set +`FUNNEL_ALLOW_INSECURE_CREDENTIALS=true` for Go clients, the CLI, and connecting processes. +Authenticated remote plaintext gRPC connections require the explicit setting +or a secure tunnel ending on loopback. The opt-in also permits authenticated +HTTPS-to-HTTP redirects. See [remote server connections](integrations/remote-server.md). + +Follow pagination tokens when listing tasks or nodes. Node listings default to +500 entries and accept page sizes from 1 through 1000. Upgrade older gRPC node +clients to use the new pagination fields. + +See [Go packages and the TES client](development/go-packages.md) for the full +package layout and client behavior. + +## Rebuild authorization plugins + +Update plugins to import `api/config`, `api/tes`, and `api/plugin`. Consolidate +the old `plugins/proto` and `plugins/shared` imports into one `api/plugin` +import. Implement `plugin.Authorize`, use `plugin.Handshake`, and choose +`plugin.AuthorizePlugin` for net/rpc or `plugin.AuthorizeGRPCPlugin` for gRPC. +Use your own logger instead of the former `plugins/shared.Logger`. + +Rebuild plugin binaries against the same Funnel module version as the upgraded +host and deploy them together. The net/rpc adapters now carry complete protobuf +messages inside Gob. This preserves optional TES fields and configuration +oneofs without global Gob registrations. The encoding differs from older +net/rpc binaries, so both peers must be rebuilt. The gRPC wire format is unchanged. +Host process loading remains internal. + +### Return the complete edited configuration + +Local and Kubernetes backends now merge plugin responses with +`WithOverwriteWithEmptyValue`. False, zero, empty strings, cleared collections, +and removed map entries can clear inherited settings. Plugins that return +partial configurations can erase required server settings. + +Clone the supplied configuration, edit that copy, and return it with the task: + +```go +package example + +import ( + "github.com/calypr/funnel/api/config" + "github.com/calypr/funnel/api/plugin" + "github.com/calypr/funnel/api/tes" + "google.golang.org/protobuf/proto" +) + +func ConfigureTask(input *config.Config, task *tes.Task) *plugin.JobResponse { + effective := proto.Clone(input).(*config.Config) + effective.Worker.LeaveWorkDir = false + return &plugin.JobResponse{Code: 200, Config: effective, Task: task} +} +``` + +Preserve settings unrelated to your changes. Kubernetes task overrides cannot +change the backend's Jobs namespace. The server validates the resulting task +configuration. See the [Go package contracts](development/go-packages.md#application-structure) +and `api/plugin/README.md` in the source checkout. + +## Migrate deployment configuration + +Remove retired keys instead of leaving unused sections in a full configuration. +The strict configuration parser rejects unknown fields. + +- Remove `Elastic`, even if another database is selected. Elasticsearch users + must migrate their data to a supported backend. +- Set `FTPStorage.Endpoint` when using configured FTP credentials. The hostname + and port must match the requested origin. See [FTP credentials](storage/ftp.md). +- For Slurm, PBS, HTCondor, and Grid Engine, set `HPCStateDir` to an absolute, + persistent controller directory with mode `0700`. Keep it outside task work + directories and enabled local-storage roots. Use a bounded + `LocalStorage.AllowedDirs` list or disable local storage. Back up this directory + with the database. See [trusted HPC submission records](compute/deployment.md#trusted-hpc-submission-records). +- Update custom runtime templates that use `index .Env "KEY"` or + `index .Tags "KEY"`. Use `.GetEnvArgs`, `.GetTagArgs`, or a `range` with `eq`. + Runtime argument placeholders must reach the runtime unchanged. See + [compute deployment templates](compute/deployment.md). + +### Replace retired Kubernetes storage settings + +If your configuration uses retired storage keys, migrate those definitions to +native objects under `Kubernetes.Storage`. +The retired names include `PVCMode`, `StorageClassName`, `SharedPVCName`, +`S3FilesFilesystemId`, `Bindings`, and `MountedInputs`. Replace +`MountedRead.WorkerMount` and `MountedWrite.WorkerMount` with `Volume`, using +`SubPath` when selecting one of several mounts of a volume. + +Use a static logical name such as `workspace` for a PVC template's +`metadata.name`. Funnel generates `-workspace`; remove old volume entries +that refer to the rendered task-specific claim name. Keep `WorkerImage` +independent of the storage definition. + +The legacy `PVTemplate`, `PVCTemplate`, and `JobTemplate` settings remain a +fallback when `Storage` is absent. Their presence alone does not require a +conversion to native storage configuration. + +For old Chicago binary configurations, export using the original schema before +converting to named configuration. Its `S3FilesFilesystemId` field number +conflicts with upstream `ForbiddenPathPrefixes`. Loading that binary directly +with the canonical schema cannot distinguish the two fields. Move the filesystem +ID into the native CSI PV template and rebuild affected plugins. + +See [Kubernetes storage](compute/kubernetes-storage.md#ownership-and-upgrades) +and [mounted inputs](storage/mounted-inputs.md) for complete migration examples. + +## Upgrade scheduler storage + +Convert standalone MongoDB deployments to a replica set or sharded cluster +before upgrading. A single-node replica set is sufficient for development. +Changing only the connection URI does not enable transactions. Follow the +[MongoDB upgrade procedure](databases/mongodb.md#upgrade-an-existing-standalone-server). + +At first startup, the upgraded scheduler imports active assignments into durable +claims. Resolve reported ownership conflicts or duplicate MongoDB IDs before +resuming submissions. Do not bypass failed migrations by running older servers +alongside the upgraded deployment. See [assignment storage upgrades](compute/deployment.md#assignment-storage-upgrade) +and [MongoDB startup migration](databases/mongodb.md#funnels-startup-migration). + +## Update development and deployment automation + +Keep provisioning scripts, service definitions, and infrastructure manifests in +operator-maintained deployment code. The bundled GCE provisioning, VM installer, +and systemd examples have been removed. Runtime compute backends and container +images remain available. The root executable and CLI remain reachable through +`go install github.com/calypr/funnel`. + +Update scripts that refer to the former repository layout: + +- Read documentation from `docs/`; Hugo configuration stays in `website/`. +- Use `tests/integration`, `tests/e2e`, and `tests/benchmarks` for the respective + suites. Shared integration configuration files live in `tests/testdata/config`. +- Run `make proto` for schema generation through the Buf configuration in `api/`. + Run direct Buf commands from `api/`. Proto imports are relative to that directory. + Update `plugins/proto/plugin.proto` to `plugin/plugin.proto`, + `events/events.proto` to `internal/events/events.proto`, and + `compute/scheduler/scheduler.proto` to `internal/scheduler/scheduler.proto`. + The TES service declaration is now in `internal/tesrpc/service.proto`; shared + TES messages remain in `tes/tes.proto`. Message namespaces and service names + remain unchanged, but file moves break Buf's `FILE` compatibility policy. +- Update custom linker flags to use `github.com/calypr/funnel/version`. +- Remove calls to the retired `bundle-examples` Make target. + +The source build currently requires Go 1.27.3, as declared in `go.mod`. +Integration tests clean up generated files by default. Pass +`-funnel-keep-artifacts` when retaining them for debugging. Resources still close +when files are retained. See [developer test instructions](development/developers.md#go-tests) +and `tests/README.md` in the source checkout. + +## Verify the upgrade + +Build and test downstream applications and rebuilt plugins against the selected +Funnel version. In a test deployment, verify authenticated task creation, +retrieval, cancellation, input downloads, output publication, and pagination. +If authentication is configured, check that unauthorized requests remain +rejected. Check that plugins preserve unrelated configuration settings when +clearing individual values. + +Confirm that startup migrations finish before resuming normal submissions. +Keep backups and older binaries until the upgraded deployment and its rollback +procedure have been verified. diff --git a/docs/tools/funnel/index.md b/docs/tools/funnel/index.md index 05176f5e..a893ef48 100644 --- a/docs/tools/funnel/index.md +++ b/docs/tools/funnel/index.md @@ -1,15 +1,14 @@ ---- -title: Funnel ---- - # Funnel -Funnel provides task execution and workflow orchestration around the GA4GH TES model. +Funnel runs batch tasks through the GA4GH Task Execution Service API. Tasks describe their input files, container commands, output destinations, and resource requirements. -It is designed to streamline distributed batch processing by exposing a simple task API and a set of components that can be adapted to different compute and storage backends. +Funnel supports local Docker, Kubernetes, cloud batch services, and HPC schedulers. Supported deployments can also request GPUs. Operators choose the compute and storage backends; clients submit tasks through the same task API. -## Read Next +## Get started -- [Architecture Overview](docs/docs.md) -- [Download & Installation](docs/download.md) +- [Download and installation](docs/download.md) +- [Overview](docs/docs.md) - [Tasks](docs/tasks.md) +- [GPU tasks](docs/compute/gpus.md) +- [Upgrade an existing deployment](docs/upgrading.md) +- [Latest release and release notes](https://github.com/calypr/funnel/releases/latest) diff --git a/docs/tools/funnel/install.sh b/docs/tools/funnel/install.sh index 8cbeb170..902e7a50 100755 --- a/docs/tools/funnel/install.sh +++ b/docs/tools/funnel/install.sh @@ -19,7 +19,7 @@ list_tags() { RELEASES_URL="https://api.github.com/repos/calypr/funnel/releases" # Get all releases and extract tag names - RELEASES_JSON=$(curl -s "$RELEASES_URL") + RELEASES_JSON=$(curl -fsSL --retry 3 --connect-timeout 15 --max-time 180 "$RELEASES_URL") echo "$RELEASES_JSON" | grep '"tag_name":' | cut -d '"' -f 4 | head -20 | while read -r tag; do echo "$tag" @@ -84,7 +84,7 @@ get_release_url() { if [ -z "$VERSION" ]; then echo "No version specified. Fetching the latest release..." RELEASE_URL="https://api.github.com/repos/calypr/funnel/releases/latest" - VERSION=$(curl -s $RELEASE_URL | grep '"tag_name":' | cut -d '"' -f 4) + VERSION=$(curl -fsSL --retry 3 --connect-timeout 15 --max-time 180 "$RELEASE_URL" | grep '"tag_name":' | cut -d '"' -f 4) else echo "Fetching release for version $VERSION..." RELEASE_URL="https://api.github.com/repos/calypr/funnel/releases/tags/$VERSION" @@ -108,7 +108,7 @@ get_os_arch() { get_assets() { # Fetch the release assets URLs - RELEASE_JSON=$(curl -s $RELEASE_URL) + RELEASE_JSON=$(curl -fsSL --retry 3 --connect-timeout 15 --max-time 180 "$RELEASE_URL") if echo "$RELEASE_JSON" | grep -q '"status": "404"'; then echo "Release $VERSION not found." @@ -139,13 +139,13 @@ download() { TAR_URL="$asset" TAR_NAME="$asset_name" echo " ➜ $TAR_NAME" - curl -L --progress-bar -o "$TAR_NAME" "$TAR_URL" + curl -fL --retry 3 --connect-timeout 15 --max-time 180 --progress-bar -o "$TAR_NAME" "$TAR_URL" # Checksums elif [[ "$asset_name" == *checksums* ]]; then CHECKSUM_FILE="$asset_name" echo " ➜ $CHECKSUM_FILE" - curl -L --progress-bar -o "$CHECKSUM_FILE" "$asset" + curl -fL --retry 3 --connect-timeout 15 --max-time 180 --progress-bar -o "$CHECKSUM_FILE" "$asset" fi done } @@ -153,8 +153,8 @@ download() { verify() { # Verify checksum echo "Verifying checksum..." - CHECKSUM_EXPECTED=$(grep $TAR_NAME $CHECKSUM_FILE | awk '{print $1}') - CHECKSUM_ACTUAL=$(shasum -a 256 $TAR_NAME | awk '{print $1}') + CHECKSUM_EXPECTED=$(grep -F -- "$TAR_NAME" "$CHECKSUM_FILE" | awk '{print $1}') + CHECKSUM_ACTUAL=$(shasum -a 256 "$TAR_NAME" | awk '{print $1}') if [ "$CHECKSUM_EXPECTED" != "$CHECKSUM_ACTUAL" ]; then echo "Checksum verification failed for $TAR_NAME. Exiting..." @@ -165,27 +165,34 @@ verify() { install() { # Extract and install the package echo "Extracting the package..." - tar -xzf $TAR_NAME + tar -xzf "$TAR_NAME" # Determine where to install the Funnel binary if [ -z "$DEST" ]; then - DEST=$HOME/.local/bin + DEST="$HOME/.local/bin" fi echo "Installing Funnel to $DEST..." - mkdir -p $DEST - mv funnel $DEST + mkdir -p "$DEST" + mv funnel "$DEST/" # Clean up - rm $TAR_NAME $CHECKSUM_FILE + rm "$TAR_NAME" "$CHECKSUM_FILE" echo "Installation successful: $DEST/funnel" echo - $DEST/funnel version + "$DEST/funnel" version echo echo "Run '$DEST/funnel --help' for more info" } main() { + # Preserve relative destinations before extracting into a private directory. + if [[ -n "$DEST" && "$DEST" != /* ]]; then + DEST="$PWD/$DEST" + fi + install_work_dir=$(mktemp -d) + trap 'rm -rf "$install_work_dir"' EXIT + cd "$install_work_dir" get_release_url get_os_arch get_assets diff --git a/scripts/sync_funnel_docs.py b/scripts/sync_funnel_docs.py new file mode 100644 index 00000000..cadfdd44 --- /dev/null +++ b/scripts/sync_funnel_docs.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +"""Copy Funnel's public Markdown documentation into the CALYPR site.""" + +import argparse +import json +import os +import re +from pathlib import Path +from urllib.parse import urlsplit, urlunsplit + +SITE = Path(__file__).resolve().parents[1] +DEST = SITE / "docs/tools/funnel/docs" +MANIFEST = SITE / "docs/tools/funnel/.funnel-docs.json" +NAV = SITE / "zensical.toml" +GROUPS = [ + ("Storage", "storage"), ("Compute", "compute"), + ("Databases", "databases"), ("Security", "security"), + ("Integrations", "integrations"), ("Events", "events"), + ("Metrics", "metrics"), ("Development", "development"), +] +SITE_OWNED = {"download.md"} + + +def destination(path: Path) -> Path: + return Path("docs.md") if path == Path("README.md") else path + + +def convert(text: str, page: Path, upstream: Path, revision: str) -> str: + if re.search(r"\{\{[<%]", text): + raise ValueError("Unconverted Hugo shortcode in " + str(page)) + source = upstream / "docs" + parent = destination(page.relative_to(source)).parent + + def link(url: str) -> str: + parsed = urlsplit(url) + if parsed.scheme or parsed.netloc or not parsed.path or url.startswith("/"): + return url + target = (page.parent / parsed.path).resolve() + if not target.is_file(): + return url + if target.is_relative_to(source): + relative = os.path.relpath(destination(target.relative_to(source)), parent) + return urlunsplit(("", "", relative, parsed.query, parsed.fragment)) + if target.is_relative_to(upstream): + repo_path = target.relative_to(upstream).as_posix() + return urlunsplit(("https", "github.com", f"/calypr/funnel/blob/{revision}/{repo_path}", parsed.query, parsed.fragment)) + return url + + text = re.sub(r"(?m)(? str: + match = re.search(r"^#\s+(.+?)\s*$", path.read_text(), re.MULTILINE) + return match[1] if match else path.stem.replace("-", " ").title() + + +def sync(upstream: Path, revision: str) -> None: + upstream = upstream.resolve() + source = upstream / "docs" + if not (source / "tasks.md").is_file() or not (source / "README.md").is_file(): + raise ValueError("Not a Funnel source checkout: docs/README.md or docs/tasks.md is missing") + files = sorted( + p for p in source.rglob("*") + if p.is_file() and p.relative_to(source).parts[0] != "internal" + and not any(part.startswith(".") for part in p.relative_to(source).parts) + ) + pages = [p for p in files if p.suffix == ".md"] + sections = {p.relative_to(source).parts[0] for p in pages if p.parent != source} + unknown = sections - {folder for _, folder in GROUPS} + if unknown: + raise ValueError("Unknown Funnel documentation sections: " + ", ".join(sorted(unknown))) + imported = {} + for path in files: + name = destination(path.relative_to(source)).as_posix() + if name in SITE_OWNED: + raise ValueError("Upstream page conflicts with site-owned content: " + name) + imported[name] = ( + convert(path.read_text(), path, upstream, revision).encode() + if path.suffix == ".md" else path.read_bytes() + ) + + nav = [" # BEGIN GENERATED FUNNEL NAV", ' { "Funnel" = [', + ' "tools/funnel/index.md",', + ' { "Overview" = "tools/funnel/docs/docs.md" },', + ' { "Download" = "tools/funnel/docs/download.md" },'] + + def entry(page: Path, indent: str) -> str: + target = "tools/funnel/docs/" + destination(page.relative_to(source)).as_posix() + return indent + "{ " + json.dumps(title(page), ensure_ascii=False) + " = " + json.dumps(target) + " }," + + indexes = {folder + ".md" for _, folder in GROUPS} + root_pages = [p for p in pages if p.parent == source and p.name not in indexes | {"README.md"}] + priority = {"tasks.md": 0, "upgrading.md": 1} + for page in sorted(root_pages, key=lambda p: (priority.get(p.name, 2), title(p).lower())): + nav.append(entry(page, " ")) + for label, folder in GROUPS: + entries = sorted((p for p in pages if p.relative_to(source).parts[0] == folder), key=lambda p: title(p).lower()) + index = source / (folder + ".md") + if not entries and not index.is_file(): + continue + nav.append(" { " + json.dumps(label) + " = [") + if index.is_file(): + nav.append(' "tools/funnel/docs/' + index.name + '",') + nav.extend(entry(p, " ") for p in entries) + nav.append(" ] },") + nav += [" ] },", " # END GENERATED FUNNEL NAV"] + config = NAV.read_text() + begin, end = " # BEGIN GENERATED FUNNEL NAV", " # END GENERATED FUNNEL NAV" + if config.count(begin) != 1 or config.count(end) != 1 or config.index(begin) >= config.index(end): + raise ValueError("Missing or invalid Funnel navigation markers") + + DEST.mkdir(parents=True, exist_ok=True) + for path in DEST.rglob("*"): + if path.is_file() and path.relative_to(DEST).as_posix() not in imported.keys() | SITE_OWNED: + path.unlink() + for name, content in imported.items(): + target = DEST / name + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(content) + (SITE / "docs/tools/funnel/install.sh").write_bytes((upstream / "install.sh").read_bytes()) + NAV.write_text(config[:config.index(begin)] + "\n".join(nav) + config[config.index(end) + len(end):]) + MANIFEST.write_text(json.dumps({ + "source": "https://github.com/calypr/funnel", "branch": "develop", + "source_directory": "docs", "revision": revision, "files": sorted(imported), + }, indent=2) + "\n") + print(f"Synced {len(pages)} Funnel pages from {revision}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("upstream", type=Path) + parser.add_argument("--revision", required=True) + args = parser.parse_args() + if not re.fullmatch(r"[a-f0-9]{40}", args.revision): + parser.error("--revision must be a full Git commit SHA") + sync(args.upstream.resolve(), args.revision) diff --git a/tests/test_sync_funnel_docs.py b/tests/test_sync_funnel_docs.py new file mode 100644 index 00000000..636dedea --- /dev/null +++ b/tests/test_sync_funnel_docs.py @@ -0,0 +1,107 @@ +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path + +SPEC = importlib.util.spec_from_file_location( + "sync_funnel_docs", Path(__file__).resolve().parents[1] / "scripts/sync_funnel_docs.py" +) +sync = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(sync) +REV = "a" * 40 + + +class SyncTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + root = Path(self.temp.name) + self.site, self.upstream = root / "site", root / "upstream" + self.source = self.upstream / "docs" + (self.source / "compute").mkdir(parents=True) + (self.site / "docs/tools/funnel/docs").mkdir(parents=True) + (self.upstream / "install.sh").write_text("#!/bin/sh\n") + (self.source / "README.md").write_text("# Overview\n") + (self.source / "tasks.md").write_text("# Tasks\n") + (self.source / "upgrading.md").write_text("# Upgrade Funnel\n") + (self.source / "compute/example.md").write_text("# GPU tasks\nnew\n") + self.nav = self.site / "zensical.toml" + self.nav.write_text(" # BEGIN GENERATED FUNNEL NAV\n # END GENERATED FUNNEL NAV\n") + sync.SITE = self.site + sync.DEST = self.site / "docs/tools/funnel/docs" + sync.MANIFEST = self.site / "docs/tools/funnel/.funnel-docs.json" + sync.NAV = self.nav + + def tearDown(self): + self.temp.cleanup() + + def test_updates_plain_markdown_navigation_and_removes_retired_pages(self): + download = sync.DEST / "download.md" + download.write_text("site-owned download page") + retired = sync.DEST / "databases/elasticsearch.md" + retired.parent.mkdir() + retired.write_text("retired") + sync.sync(self.upstream, REV) + self.assertFalse(retired.exists()) + self.assertEqual(download.read_text(), "site-owned download page") + self.assertIn('"GPU tasks"', self.nav.read_text()) + self.assertIn('"Upgrade Funnel"', self.nav.read_text()) + self.assertEqual((sync.DEST / "docs.md").read_text(), "# Overview\n") + previous = {p.relative_to(self.site): p.read_bytes() for p in self.site.rglob("*") if p.is_file()} + sync.sync(self.upstream, REV) + self.assertEqual(previous, {p.relative_to(self.site): p.read_bytes() for p in self.site.rglob("*") if p.is_file()}) + (self.source / "compute/example.md").unlink() + sync.sync(self.upstream, REV) + self.assertFalse((sync.DEST / "compute/example.md").exists()) + self.assertNotIn("compute/example.md", self.nav.read_text()) + + def test_adopts_reconciled_docs_instead_of_keeping_legacy_hash_locks(self): + sync.MANIFEST.write_text(json.dumps({"files": [], "pending_reconciliation": {"compute/example.md": "old-hugo-hash"}})) + sync.sync(self.upstream, REV) + self.assertIn("new", (sync.DEST / "compute/example.md").read_text()) + self.assertNotIn("pending_reconciliation", json.loads(sync.MANIFEST.read_text())) + + def test_preserves_reference_assets_and_rewrites_repository_links(self): + templates = self.source / "compute/templates" + templates.mkdir() + (templates / "role.yaml").write_text("kind: Role\n") + (self.upstream / "internal").mkdir() + (self.upstream / "internal/example.go").write_text("package example\n") + (self.source / "compute/example.md").write_text( + "# GPU tasks\n[Overview](../README.md)\n[Template](templates/role.yaml)\n" + "[Code](../../internal/example.go#L1)\n[code]: ../../internal/example.go\n" + ) + sync.sync(self.upstream, REV) + text = (sync.DEST / "compute/example.md").read_text() + self.assertIn("[Overview](../docs.md)", text) + self.assertIn("[Template](templates/role.yaml)", text) + self.assertIn(f"https://github.com/calypr/funnel/blob/{REV}/internal/example.go#L1", text) + self.assertIn(f"[code]: https://github.com/calypr/funnel/blob/{REV}/internal/example.go", text) + self.assertEqual((sync.DEST / "compute/templates/role.yaml").read_text(), "kind: Role\n") + + def test_excludes_private_internal_and_hidden_files(self): + (self.source / "internal").mkdir() + (self.source / "internal/private-plan.md").write_text("private") + (self.source / ".private.md").write_text("private") + sync.sync(self.upstream, REV) + self.assertFalse((sync.DEST / "internal").exists()) + self.assertFalse((sync.DEST / ".private.md").exists()) + self.assertNotIn("private", self.nav.read_text()) + + def test_rejects_unknown_sections_before_modifying_site(self): + (self.source / "surprise").mkdir() + (self.source / "surprise/page.md").write_text("# Surprise\n") + original = self.nav.read_text() + with self.assertRaisesRegex(ValueError, "Unknown Funnel documentation sections"): + sync.sync(self.upstream, REV) + self.assertEqual(original, self.nav.read_text()) + + def test_rejects_unknown_shortcodes_before_modifying_site(self): + (self.source / "compute/example.md").write_text("{{< surprise >}}") + with self.assertRaisesRegex(ValueError, "Unconverted Hugo shortcode"): + sync.sync(self.upstream, REV) + self.assertFalse((sync.DEST / "tasks.md").exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/zensical.toml b/zensical.toml index 4651955e..c902ac3a 100644 --- a/zensical.toml +++ b/zensical.toml @@ -93,56 +93,75 @@ nav = [ ] }, ] }, { "Workflow Execution" = [ + # BEGIN GENERATED FUNNEL NAV { "Funnel" = [ "tools/funnel/index.md", { "Overview" = "tools/funnel/docs/docs.md" }, { "Download" = "tools/funnel/docs/download.md" }, { "Tasks" = "tools/funnel/docs/tasks.md" }, + { "Upgrade Funnel" = "tools/funnel/docs/upgrading.md" }, { "Storage" = [ - "tools/funnel/docs/storage/index.md", - { "Local" = "tools/funnel/docs/storage/local.md" }, - { "S3" = "tools/funnel/docs/storage/s3.md" }, + "tools/funnel/docs/storage.md", + { "FTP" = "tools/funnel/docs/storage/ftp.md" }, { "Google Storage" = "tools/funnel/docs/storage/google-storage.md" }, - { "OpenStack Swift" = "tools/funnel/docs/storage/swift.md" }, { "HTTP(S)" = "tools/funnel/docs/storage/http.md" }, - { "FTP" = "tools/funnel/docs/storage/ftp.md" }, + { "Local" = "tools/funnel/docs/storage/local.md" }, + { "Mounted inputs" = "tools/funnel/docs/storage/mounted-inputs.md" }, + { "OpenStack Swift" = "tools/funnel/docs/storage/swift.md" }, + { "S3" = "tools/funnel/docs/storage/s3.md" }, ] }, { "Compute" = [ - { "Deploying a Cluster" = "tools/funnel/docs/compute/deployment.md" }, + "tools/funnel/docs/compute.md", { "AWS Batch" = "tools/funnel/docs/compute/aws-batch.md" }, - { "GCP Batch" = "tools/funnel/docs/compute/gcp-batch.md" }, - { "Kubernetes" = "tools/funnel/docs/compute/kubernetes.md" }, + { "Deploying a cluster" = "tools/funnel/docs/compute/deployment.md" }, + { "GPU tasks" = "tools/funnel/docs/compute/gpus.md" }, + { "Grid Engine" = "tools/funnel/docs/compute/grid-engine.md" }, { "HTCondor" = "tools/funnel/docs/compute/htcondor.md" }, - { "Slurm" = "tools/funnel/docs/compute/slurm.md" }, + { "Kubernetes" = "tools/funnel/docs/compute/kubernetes.md" }, + { "Kubernetes reference templates" = "tools/funnel/docs/compute/kubernetes-templates/README.md" }, + { "Kubernetes storage" = "tools/funnel/docs/compute/kubernetes-storage.md" }, + { "Overview" = "tools/funnel/docs/compute/gcp-batch.md" }, { "PBS/Torque" = "tools/funnel/docs/compute/pbs-torque.md" }, - { "Grid Engine" = "tools/funnel/docs/compute/grid-engine.md" }, + { "Slurm" = "tools/funnel/docs/compute/slurm.md" }, ] }, { "Databases" = [ + "tools/funnel/docs/databases.md", + { "Badger" = "tools/funnel/docs/databases/badger.md" }, + { "DynamoDB" = "tools/funnel/docs/databases/dynamodb.md" }, { "Embedded" = "tools/funnel/docs/databases/boltdb.md" }, + { "Google Cloud Datastore" = "tools/funnel/docs/databases/datastore.md" }, { "MongoDB" = "tools/funnel/docs/databases/mongodb.md" }, - { "Elasticsearch" = "tools/funnel/docs/databases/elasticsearch.md" }, - { "DynamoDB" = "tools/funnel/docs/databases/dynamodb.md" }, - { "Datastore" = "tools/funnel/docs/databases/datastore.md" }, + { "Postgres" = "tools/funnel/docs/databases/postgres.md" }, ] }, { "Security" = [ + "tools/funnel/docs/security.md", { "Basic Auth" = "tools/funnel/docs/security/basic.md" }, { "OAuth2" = "tools/funnel/docs/security/oauth2.md" }, - { "Advanced Auth" = "tools/funnel/docs/security/advanced.md" }, + { "Overview 🔐" = "tools/funnel/docs/security/advanced.md" }, ] }, { "Integrations" = [ + { "Connect to a remote Funnel server" = "tools/funnel/docs/integrations/remote-server.md" }, { "Nextflow" = "tools/funnel/docs/integrations/nextflow.md" }, { "py-tes" = "tools/funnel/docs/integrations/py-tes.md" }, ] }, { "Events" = [ + "tools/funnel/docs/events.md", + { "Event files" = "tools/funnel/docs/events/file.md" }, { "Kafka" = "tools/funnel/docs/events/kafka.md" }, ] }, { "Metrics" = [ + "tools/funnel/docs/metrics.md", { "Prometheus" = "tools/funnel/docs/metrics/prometheus.md" }, ] }, { "Development" = [ - { "Funnel Developers" = "tools/funnel/docs/development/developers.md" }, + "tools/funnel/docs/development.md", + { "Continuous integration" = "tools/funnel/docs/development/ci.md" }, + { "Developers" = "tools/funnel/docs/development/developers.md" }, + { "Go packages and the TES client" = "tools/funnel/docs/development/go-packages.md" }, + { "Troubleshooting information" = "tools/funnel/docs/development/diagnostics.md" }, ] }, ] }, + # END GENERATED FUNNEL NAV ] }, { "Graph Database" = [ { "GRIP" = [