diff --git a/src/.vitepress/routes/sidebar/index.ts b/src/.vitepress/routes/sidebar/index.ts index e4ee6bf3..3f7f2643 100644 --- a/src/.vitepress/routes/sidebar/index.ts +++ b/src/.vitepress/routes/sidebar/index.ts @@ -13,7 +13,7 @@ export const sidebarRoutes = [ {text: 'Architecture', link: '/architecture'}, {text: 'Components', link: '/components'}, {text: 'Guides', link: '/guides'}, - {text: 'Glossar', link: '/glossar'}, + {text: 'Glossary', link: '/glossar'}, {text: 'FAQs', base:'/guide/faqs', collapsed: true, items: faqsRoutes}, ] }, diff --git a/src/.vitepress/routes/sidebar/user.ts b/src/.vitepress/routes/sidebar/user.ts index 1c96c1e6..a71ee180 100644 --- a/src/.vitepress/routes/sidebar/user.ts +++ b/src/.vitepress/routes/sidebar/user.ts @@ -15,6 +15,7 @@ export const userRoutes = [ text: 'Concepts/Tutorials', collapsed: true, items: [ + { text: 'Developing with the SDK', link: '/sdk-how-to' }, { text: 'Analysis Coding', collapsed: true, @@ -57,6 +58,7 @@ export const userRoutes = [ ] }, { text: 'FHIR Queries', link: '/fhir-query' }, + { text: 'Master Images', link: '/master-images' }, // {text: 'Homomorphic Encryption', link: '/homomorphic-encryption'}, ] }, diff --git a/src/components/VPTeam.vue b/src/components/VPTeam.vue index e6ef428a..5caeb654 100644 --- a/src/components/VPTeam.vue +++ b/src/components/VPTeam.vue @@ -74,14 +74,6 @@ const members = [ { icon: 'github', link: 'https://github.com/pbrassel' }, ], }, - { - avatar: 'https://github.com/AbuAttieh.png', - name: 'Hammam Abu Attieh', - title: 'Privacy Engineer ', - links: [ - { icon: 'github', link: 'https://github.com/AbuAttieh' }, - ], - }, { avatar: 'https://github.com/mhalilovic.png', name: 'Mehmed Halilovic', @@ -181,6 +173,14 @@ const alumni = [ { icon: 'github', link: 'https://github.com/raphaelre' }, ], }, + { + avatar: 'https://github.com/AbuAttieh.png', + name: 'Hammam Abu Attieh', + title: 'Privacy Engineer ', + links: [ + { icon: 'github', link: 'https://github.com/AbuAttieh' }, + ], + }, ]; const pi = [ diff --git a/src/getting-started/architecture.md b/src/getting-started/architecture.md index 7e2d34a1..0a7f448a 100644 --- a/src/getting-started/architecture.md +++ b/src/getting-started/architecture.md @@ -8,7 +8,93 @@ const image = { # Architecture -This graphic is intended to provide a rough overview of the FLAME execution flow. -On the next page more detailed information about the individual services is given. +FLAME consists of two parts: one central **Hub** and one **Node** at every participating site. The Hub coordinates, +the Nodes compute. Patient data stays at the site, only analysis code travels to the data and only aggregated results +travel back. + +The graphic gives a rough overview of the execution flow of an analysis. The individual services are listed on the +[Components](./components) page. + +## Hub and Node + +| | Hub | Node | +|---|---|---| +| Runs | once, centrally | once per site (hospital, university, ...) | +| Purpose | coordination | execution on local data | +| Holds | users, projects, analysis code, analysis images, results | the site's data stores and running analyses | +| Sees patient data | no | yes, inside the site only | +| Used by | researchers and administrators | node administrators and data stewards | + +### Hub + +The Hub is the single place researchers work with. Its services: + +* **User Interface** to create projects and analyses, review them and download results. +* **Core** as the main backend, managing all resources and triggering commands and events. +* **Authentication and authorization** (Authup), organized in one *realm* per organization. +* **Analysis building**, which combines the approved code with a master image into a container image. +* **Container image registry** (Harbor) with a separate, private project for every node. +* **Message Broker**, which relays the messages nodes send to each other. +* **Storage** for code, intermediate results and final results. + +### Node + +A Node is installed on a Kubernetes cluster inside the network of a site. Its services: + +* **Node UI** to configure data stores and to start, stop and monitor analyses. +* **Hub API Adapter**, the backend of the Node UI and the connection to the Hub. +* **Pod Orchestration**, which starts every analysis in its own isolated pod. +* **Data access through an API gateway** (Kong), which exposes exactly the data stores of a project to its analyses. +* **Message Broker** and **Result Service**, the node side of the communication with other nodes and the Hub. + +## Lifecycle of an analysis + +1. **Project:** a researcher creates a project in the Hub and selects the nodes that should take part. Every selected + node approves or rejects the project. +2. **Data stores:** at every approving site, an administrator or data steward connects the requested data (a FHIR + server or an S3 bucket) to the project as a data store. +3. **Analysis:** the researcher creates an analysis within the project, uploads the code and selects a master image + with the required dependencies. +4. **Review:** the administrator of every node can download and read the code, and approves or rejects the analysis. +5. **Build and distribution:** after the approvals, the Hub builds one container image from master image and code and + pushes it into the registry project of each node. +6. **Execution:** every node pulls its image. The node administrator starts the analysis in the Node UI. The analysis + runs in an isolated pod and can only reach the data stores of its own project. +7. **Aggregation:** the analyzing nodes send their results to the node with the aggregator role, which combines them. + For iterative methods such as federated learning this repeats over several rounds. +8. **Results:** the aggregator submits the final result to the Hub storage, where the researcher downloads it. + +The same steps from the perspective of each role are described in the [User](../guide/user/index) and +[Admin](../guide/admin/index) guides. + +## Communication + +* **Nodes connect to the Hub, never the other way round.** All connections are initiated by the Node as outbound + HTTPS. A Node does not have to be reachable from the internet. +* **Nodes do not talk to each other directly.** Messages and intermediate results are relayed through the Hub. +* **What is relayed is encrypted end to end.** Messages and intermediate results are encrypted for the receiving + nodes with the node key pairs, so the Hub cannot read them. +* **Analyses are isolated.** An analysis container has no internet access and no access to other services of the + Node. It only reaches its project's data, the message broker and the result service through a dedicated proxy. + +Details are documented in [Security & Operations](../guide/deployment/node-security). + +## Where data lives + +| Data | Location | Leaves the site | +|---|---|---| +| Patient data (FHIR resources, files) | data stores of the site | no | +| Analysis code | Hub storage, and inside the analysis image | yes, it is distributed to the nodes | +| Analysis image | Hub registry, pulled by the nodes | yes, it is distributed to the nodes | +| Messages and intermediate results | relayed and stored by the Hub, end-to-end encrypted | yes, readable only by the receiving nodes | +| Local intermediate results and checkpoints | storage of the node | no | +| Final results | Hub storage | yes, downloadable by the researcher | + +Because final results leave the federation, an analysis must only return aggregated, non-identifying values. This is +what the code review by the node administrators is for. + +## Deployment + +Hub and Node are both deployed with Helm charts on Kubernetes. See the [Deployment guide](../guide/deployment/index). diff --git a/src/getting-started/glossar.md b/src/getting-started/glossar.md index 428d467b..a7a3cbc6 100644 --- a/src/getting-started/glossar.md +++ b/src/getting-started/glossar.md @@ -1,22 +1,47 @@ +# Glossary - -| Term | Explanation | -|:--------------:|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:| -| Aggregation protocol | Algorithm code how final/online results are computed| -| Analysis | Docker containers containing algorithm, analysis logic and software dependencies | -| Data discovery | Pre-analysis to verify biases, schemas or any other unexpected behavior by sites | -| Federated Learning | Parallel execution of analysis code at multiple sites, at the same time with a global result aggregation | -| FHIR | Fast Healthcare Interoperability Resources | -| FHIR Core | Common data schema developed by MII to be fulfilled by each DIC | -| Final results | Results achieved by aggregator, can be encrypted project specific | -| FLAME | Federated Learning and Analysis In Medicine | -| Hub | All service components centrally hosted and operated | -| UI | User Interface | -| Paillier - key | This is the key to a homomorphic encryption system to additionally secure the calculated data inside the analysis | -| Proposal | A Proposal is an organizational unit in FLAME, which represents the collaboration between different participants in regard to a specific research or analysis project. It contains an initial risk assessment as well as a high level description of the requested data. | -| Registry | Service to distribute analysis | -| Result inspection | Before results are returned these should be approved by sites to not contain unauthorized information | -| RSA - key | This is the key to a homomorphic encryption system to secure the analysis from external influences | -| Nodes | Hospitals hosting a node to provide analysis controlled and secure access to patient data | - - +| Term | Explanation | +|:---|:---| +| Aggregation protocol | Algorithm code how final/online results are computed | +| Aggregator | Node of an analysis that combines the results of the analyzers, decides in iterative analyses whether another round is needed, and submits the final result to the Hub | +| Analysis | Docker containers containing algorithm, analysis logic and software dependencies | +| Analyzer | Node of an analysis that executes the analysis code on its local data and sends its result to the aggregator | +| Authup | Identity and access management service of the Hub, managing realms, users, roles, clients and permissions | +| Bucket | Storage location for files in an S3 object store. A bucket is connected to a project as a data store | +| Checkpoint | Saved state of an analysis in the local storage of a node, from which a later analysis of the same project can resume | +| Client | Technical account for automated access to the Hub API, which cannot log in to the user interface (formerly *robot*) | +| Core SDK | Python library (`FlameCoreSDK`) through which analysis code accesses data, exchanges messages with other nodes and submits results | +| Data discovery | Pre-analysis to verify biases, schemas or any other unexpected behavior by sites | +| Data steward | Role at a node that may create and modify data stores, but not start or stop analyses | +| Data store | Configuration at a node that maps a project to a data source of the site, either a FHIR server or an S3 bucket | +| DIC | Data Integration Center. Unit at a university hospital of the MII that provides clinical data for research | +| Differential privacy | Technique that adds calibrated noise to a result, so that it reveals less about individual persons | +| Federated Learning | Parallel execution of analysis code at multiple sites, at the same time with a global result aggregation | +| FHIR | Fast Healthcare Interoperability Resources | +| FHIR Core | Common data schema developed by MII to be fulfilled by each DIC | +| Final results | Results achieved by aggregator, can be encrypted project specific | +| FLAME | Federated Learning and Analysis In Medicine | +| Harbor | Container image registry of the Hub. Analysis images are distributed to the nodes through a separate project per node | +| Homomorphic encryption | Encryption scheme that allows computing on encrypted values without decrypting them | +| Hub | All service components centrally hosted and operated | +| Identity provider | External login service (OIDC) that can be connected to a realm, so users log in with their existing account | +| Intermediate results | Results stored during an analysis, either locally on a node or on the Hub, encrypted for the receiving nodes | +| Kong | API gateway of a node through which an analysis accesses the data stores of its project | +| Master image | Base container image providing the runtime and software dependencies on which an analysis is built | +| Message broker | Service that relays messages between the nodes of an analysis through the Hub | +| MII | Medical Informatics Initiative. German funding program in which PrivateAIM is a project | +| Nodes | Hospitals hosting a node to provide analysis controlled and secure access to patient data | +| Paillier - key | This is the key to a homomorphic encryption system to additionally secure the calculated data inside the analysis | +| Permission | Predefined action in the Hub that is granted to users through roles | +| PrivateAIM | Project within the MII that develops the FLAME platform | +| Project | Current name of a proposal in the Hub user interface, see *Proposal* | +| Proposal | A Proposal is an organizational unit in FLAME, which represents the collaboration between different participants in regard to a specific research or analysis project. It contains an initial risk assessment as well as a high level description of the requested data. | +| Proxy | Node of an analysis without data that pre-aggregates the results of its assigned analyzers, so the aggregator never sees the result of a single node (`ProxyModel`) | +| Realm | Administrative namespace in the Hub representing one organization, with its own users, roles, identity providers and nodes | +| Registry | Service to distribute analysis | +| Result inspection | Before results are returned these should be approved by sites to not contain unauthorized information | +| Role | Named set of permissions that is assigned to users | +| RSA - key | This is the key to a homomorphic encryption system to secure the analysis from external influences | +| S3 | Object storage interface. FLAME reads files such as CSV or VCF from S3 buckets | +| Star pattern | Analysis topology in which every analyzer reports to one aggregator (`StarModel`) | +| UI | User Interface | diff --git a/src/getting-started/index.md b/src/getting-started/index.md index 1836c6c5..0dc942ba 100644 --- a/src/getting-started/index.md +++ b/src/getting-started/index.md @@ -17,6 +17,40 @@ With its innovative approach, PrivateAIM is poised to drive significant advancem improve patient care outcomes. +## FLAME at a Glance + +FLAME (Federated Learning and Analysis in Medicine) is the platform developed by PrivateAIM. It lets researchers +analyze medical data from several institutions without the data ever leaving them. + +* **The data stays where it is.** Every participating site runs a *Node* next to its own data. Instead of collecting + data centrally, FLAME sends the analysis code to the data. +* **A central Hub coordinates.** Researchers create projects, submit analyses and download results in the *Hub*. The + Hub never has access to patient data. +* **Sites keep control.** Every site decides whether it takes part in a project, reviews the analysis code before it + runs, and starts the execution itself. +* **Only aggregated results are returned.** The results of the sites are combined by an aggregator, and only this + combined result is made available to the researcher. + +How these parts work together is explained in the [Architecture](./architecture). + +### What you can do with FLAME + +* Run descriptive statistics and cohort counts across sites, on FHIR data or on files such as CSV or VCF. +* Train machine learning models with federated learning over several rounds. +* Use privacy-enhancing techniques such as local differential privacy. +* Bring your own Python analysis, built on the FLAME SDK and a master image with your dependencies. + +### Where to start + +| You are | You want to | Start here | +|---|---|---| +| a researcher or analyst | run an analysis on data of several sites | [User guide](../guide/user/index) | +| a node or realm administrator | manage users, review projects and analyses, provide data | [Admin guide](../guide/admin/index) | +| an operator | install a Hub or a Node | [Deployment guide](../guide/deployment/index) | +| new to the terminology | look up a term | [Glossary](./glossar) | + +Common questions are answered in the [FAQs](../guide/faqs/faqs-of-getting-started). + [//]: # ([![Overview](/images/process_images/pht_services.png)](/images/process_images/pht_services.png)) ## Mission Statement @@ -41,16 +75,18 @@ We provide users with an intuitive experience while driving progress in medical ## Security -### Security Protocol - -The following (WIP) flow chart depicts the security protocol used for protecting participating nodes against malicious -code, -as well as encrypting any stored results using envelope encryption. -This ensures that only approved algorithms are executed and that only previously registered participants in an -analysis can access the results. +FLAME protects the data of the participating sites on several levels: +* **Data locality:** patient data is never transferred to the Hub or to other sites. +* **Code review:** an analysis is only built and executed after every participating node has approved its code. +* **Isolation:** every analysis runs in its own container without internet access, and can only reach the data stores + of its own project. +* **Encryption:** all connections use TLS. Messages and intermediate results exchanged between nodes are additionally + encrypted end to end. +* **Access control:** users, roles and permissions are managed per organization. Starting an analysis and configuring + data stores is reserved for the staff of the site. -[//]: # ([![Security Protocol](/images/process_images/security_protocol.png)](/images/process_images/security_protocol.png)) +The technical details are documented in [Security & Operations](../guide/deployment/node-security). ## Terms of Use diff --git a/src/guide/admin/bucket-setup-for-data-store.md b/src/guide/admin/bucket-setup-for-data-store.md index 4096a5ec..cabd0bb7 100644 --- a/src/guide/admin/bucket-setup-for-data-store.md +++ b/src/guide/admin/bucket-setup-for-data-store.md @@ -22,6 +22,12 @@ Enter a username and (optionally) an email address for the new user, then assign access. Users with the "Admin (Full Access)" permission can modify and add anything they want. For executing analyses, the "Read" and "List" permissions are required. +::: warning Important +The user whose access and secret keys are used for the data store needs **both** the "List" and the "Read" permission +on the bucket. "List" is required to find the files in the bucket, "Read" to retrieve their content. If one of them is +missing, analyses cannot access the data. +::: + ### Access and Secret Keys When the user is created, they will have a pair of access and secret keys generated for them, and they will be shown on diff --git a/src/guide/user/analysis.md b/src/guide/user/analysis.md index 8ef481c8..fe477c2c 100644 --- a/src/guide/user/analysis.md +++ b/src/guide/user/analysis.md @@ -15,7 +15,7 @@ Click **+ Add** to open the analysis creation form. Here you can optionally prov Analysis Wizard — key steps: - Node Selection: In the **Nodes** tab, choose the nodes the analysis should run on. By default, all nodes available in the project are selected. - Upload: In the **Code** tab, upload one or more analysis code files — either a single file or an entire directory. - - Image Selection: In the **Image** tab, define the master image group and select the appropriate base image (e.g., Python base, ML, etc.). + - Image Selection: In the **Image** tab, define the master image group and select the appropriate base image (e.g., Python base, ML, etc.). If no image contains the dependencies you need, see [Requesting a Master Image](./master-images). - Select Entrypoint: In the **Image** tab, select the file that should serve as the analysis's entrypoint. - Lock Configuration: Back on the **Overview** tab, once all requirements are fulfilled, the analysis must be locked to proceed to the next step. diff --git a/src/guide/user/index.md b/src/guide/user/index.md index 5d11e349..3c6412bc 100644 --- a/src/guide/user/index.md +++ b/src/guide/user/index.md @@ -26,6 +26,7 @@ integrity of the original data. **`Concepts/Tutorials`** +- [Developing an Analysis with the SDK](/guide/user/sdk-how-to) - [Coding an Analysis](/guide/user/analysis-coding) - [Coding an Analysis with the Proxy Pattern](/guide/user/proxy-analysis-coding) - Example Analyses @@ -45,6 +46,7 @@ integrity of the original data. - [Utilizing Differential Privacy for Privacy Enhancement](/guide/user/testing_examples/local-testing-dp-example) - [Analysing longitudinal Long COVID (PASC) data](/guide/user/testing_examples/local-testing-pasc-example) - [FHIR Queries](/guide/user/fhir-query) +- [Requesting a Master Image](/guide/user/master-images) **`Using the Hub`** diff --git a/src/guide/user/master-images.md b/src/guide/user/master-images.md new file mode 100644 index 00000000..0cf04e84 --- /dev/null +++ b/src/guide/user/master-images.md @@ -0,0 +1,201 @@ +# Requesting a Master Image + +A **master image** is the container image your analysis code is built on. It provides the runtime (e.g. Python) and +all software dependencies. Packages cannot be installed while an analysis runs, so every library your script imports +has to be part of the master image you select when [creating the analysis](/guide/user/analysis). + +All master images are maintained in the public +[PrivateAIM/master-images](https://github.com/PrivateAIM/master-images) repository. If none of them fits, you request +a new one with a pull request. This page describes that process: + +1. [Check whether a suitable image already exists](#_1-check-the-existing-master-images) +2. [Create your own master image](#_2-create-your-master-image) +3. [Test it locally](#_3-test-the-image-locally) +4. [Open a pull request](#_4-open-a-pull-request) +5. [Synchronize the Hub](#_5-synchronize-the-hub) + +## 1. Check the existing master images + +Please check first whether a master image already contains the dependencies you need. Every image is a folder below +[`data/`](https://github.com/PrivateAIM/master-images/tree/master/data) in the repository, and its +`requirements.txt` lists the installed packages. + +| Location | Contains | Examples | +|---|---|---| +| `data/python/` | general purpose Python images | `base`, `ml`, `pytorch`, `tensorflow`, `genomics` | +| `data/python/use-cases/` | images built for a specific project or analysis | `fedstats`, `gemtex`, `record-linkage` | +| `data/python/conda/` | images based on a conda environment | `fed-gwas` | +| `data/r/` | R images | `ml` | + +The same images are offered in the Hub in the **Image** tab of the analysis wizard. + +If an existing image only lacks one or two packages, consider proposing to extend it instead of adding a new one. + +## 2. Create your master image + +Create a new branch in the [master-images](https://github.com/PrivateAIM/master-images) repository. If you do not +have write access, fork the repository first. + +```shell +git clone https://github.com/PrivateAIM/master-images.git +cd master-images +git checkout -b add-my-analysis-image +``` + +Then: + +1. **Create a new folder** named after your master image, e.g. `data/python/use-cases/my-analysis/`. The folder name + becomes the name of the image. +2. **Copy the `Dockerfile` and `requirements.txt`** of one of the examples into it, e.g. from `fedstats`, `gemtex` or + `pytorch`. Pick the one closest to what you need: `fedstats` and `gemtex` start from a plain Python image, `pytorch` + from an NVIDIA CUDA image for GPU workloads. +3. **Adjust `requirements.txt`** to the dependencies your analysis script requires. + +```shell +mkdir data/python/use-cases/my-analysis +cp data/python/use-cases/fedstats/Dockerfile \ + data/python/use-cases/fedstats/requirements.txt \ + data/python/use-cases/my-analysis/ +``` + +The resulting folder looks like this: + +``` +data/python/use-cases/my-analysis/ +├── Dockerfile +├── requirements.txt +└── README.md +``` + +The `README.md` is optional and holds one or two sentences on what the image is for. + +`requirements.txt`: + +``` +pandas == 2.2.3 +numpy == 1.26.4 +scikit-learn == 1.6.1 +``` + +`Dockerfile` (copied from `fedstats`): + +```dockerfile +FROM python:3.13 +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get update -yqq && \ + apt-get dist-upgrade -yqq && \ + apt-get install -yqq git + +COPY requirements.txt /tmp/requirements.txt + +RUN python3 -m pip install --upgrade pip + +RUN pip3 install -r /tmp/requirements.txt + +RUN pip3 install \ + git+https://github.com/PrivateAIM/python-sdk-patterns.git@1.0.0 +``` + +A few rules: + +* **Keep the last instruction.** It installs the FLAME SDK and patterns (`flame`). Without it your analysis cannot start. + Use the same version as the example you copied from. +* **Pin every version** (`package == x.y.z`), so the image builds reproducibly and can be reviewed. +* **Do not list `flamesdk` dependencies in conflicting versions.** The SDK requires specific ranges of `httpx`, + `fastapi` and `uvicorn`. If you need them, copy the versions from the example. +* **System packages or command line tools** are added with a further `apt-get install` in the `Dockerfile`. +* **No data, credentials or analysis code** belong in the image. The repository and the images are public, and your + code is added later by the Hub. + +The image group (`python`) and the start command (`python -u`) are inherited from the `image-group.json` of the parent +folder, so you do not need to configure them. + +## 3. Test the image locally + +Test before you open the pull request. A failed build, or an import error that only shows up on a node, costs +everybody a full review and approval round. You need [Docker](https://docs.docker.com/get-docker/). + +**Build the image:** + +```shell +docker build -t my-analysis:test data/python/use-cases/my-analysis +``` + +**Check that your dependencies are importable**, including `flame`: + +```shell +docker run --rm my-analysis:test \ + python -c "import flame, pandas, sklearn; print('ok')" +``` + +**Run your analysis in the image.** Write a local test script for your analysis with `StarModelTester` as described in +[Local Testing](/guide/user/local-testing), then execute it inside the container instead of your own Python +environment: + +```shell +docker run --rm -v "$PWD":/work -w /work my-analysis:test \ + python -u my_analysis_test.py +``` + +Run this from the folder that contains your script and test data. If it runs through, your script works with exactly +the package versions the nodes will use. + +::: tip +The repository also ships a small CLI that builds an image the same way the CI does: + +```shell +npm ci && npm run build +npm run cli -- list +npm run cli -- build python/use-cases/my-analysis +``` +::: + +## 4. Open a pull request + +Commit the new folder, push your branch and open a pull request against the `master` branch of +[PrivateAIM/master-images](https://github.com/PrivateAIM/master-images/pulls). + +```shell +git add data/python/use-cases/my-analysis +git commit -m "feat: add my-analysis master image" +git push -u origin add-my-analysis-image +``` + +Describe in the pull request which analysis or project the image is for and why no existing image fits. A maintainer +reviews the change. + +After the pull request has been approved and merged, a worker automatically builds the new master image and pushes it +to the Hub's Harbor registry. You do not have to build or push anything yourself. + +## 5. Synchronize the Hub + +The Hub has to learn about the new image once: + +1. In the Hub UI, go to **Admin → General → Services → Master Images**. +2. Press the green **Sync** button. + +After a few minutes, the newly built master image is available for selection in projects and analyses. + +::: info +Synchronizing requires the `MASTER_IMAGE_MANAGE` permission. If you do not see the page or the button, ask a Hub +administrator to trigger the synchronization for you. +::: + +You can now select the image in the **Image** tab when +[creating an analysis](/guide/user/analysis#_3-configure-analysis). + +## Updating an existing master image + +To add or upgrade a package in an image that already exists, follow the same steps: change its `requirements.txt` or +`Dockerfile` on a branch, test locally, open a pull request and synchronize the Hub after the merge. Keep in mind that +other analyses may use the same image. + +## Troubleshooting + +| Problem | Likely cause | What to do | +|---|---|---| +| `docker build` fails in `pip3 install -r` | version does not exist, or conflicts with another package | fix the pin, and check the Python version in `FROM` | +| `ModuleNotFoundError: flame` | the `python-sdk-patterns` line was removed | restore the last instruction of the `Dockerfile` | +| Image does not appear in the Hub | not synchronized yet, or the build is still running | press **Sync** again after a few minutes | +| Analysis fails with `ModuleNotFoundError` on the node | package missing in `requirements.txt` | add it, and repeat the local test | diff --git a/src/guide/user/sdk-how-to.md b/src/guide/user/sdk-how-to.md new file mode 100644 index 00000000..a47a984a --- /dev/null +++ b/src/guide/user/sdk-how-to.md @@ -0,0 +1,459 @@ +# Developing an Analysis with the SDK + +This page answers the questions that come up when you write an analysis: how to get your data, how to compute on it +at the node, how nodes talk to each other, and how to get results. It is task-oriented and links into the +[Python Core SDK reference](/guide/user/sdk-core-doc) for the full method signatures. + +## Two ways to write an analysis + +| | Pattern (`StarModel`, `ProxyModel`) | Core SDK (`FlameCoreSDK`) | +|---|---|---| +| You write | `analysis_method()`, `aggregation_method()`, `has_converged()` | the complete control flow of every node | +| Data retrieval | done for you, handed over as `data` | you call `get_fhir_data()` / `get_s3_data()` | +| Communication | done for you | you call `send_message()`, `await_messages()`, ... | +| Result submission | done for you | you call `submit_final_result()` | +| Use it when | analyzers compute, one aggregator combines (optionally over several rounds) | your protocol does not fit that shape | + +Start with a pattern. Inside a pattern class the SDK is still available as `self.flame`, so you can mix both, for +example to log, report progress or fetch additional data. + +* [Coding an Analysis](/guide/user/analysis-coding) introduces `StarModel`. +* [Proxy Pattern](/guide/user/proxy-analysis-coding) introduces `ProxyModel`. + +## 1. How do I get my data? + +An analysis can only reach the data stores that the node administrator attached to **its own project** +(see [Data Store Management](/guide/admin/data-store-management)). There are two kinds of data store: + +| Data store | Holds | `data_type` | You select data with | +|---|---|---|---| +| FHIR server | structured clinical resources | `'fhir'` | FHIR search queries | +| S3 bucket | files of any format (CSV, VCF, images, text, ...) | `'s3'` | object keys, i.e. file names | + +There is no direct file system access to the node. Files are made available by uploading them to a bucket, see +[Bucket Setup for Data Store](/guide/admin/bucket-setup-for-data-store). + +### With a pattern + +Set `data_type` and `query` on the model. The pattern fetches the data on every analyzer node and passes it to +`analysis_method()` as `data`: + +```python +StarModel( + analyzer=MyAnalyzer, + aggregator=MyAggregator, + # 's3' or 'fhir' + data_type='s3', + # object keys for s3, FHIR queries for fhir + query=['cohort.csv'], + ... +) +``` + +`data` is a list with one dictionary per data store of the project on that node (often exactly one). Each dictionary +maps the query to its result: + +```python +# data_type='fhir', query='Patient?_summary=count' +data = [{'Patient?_summary=count': {...FHIR response...}}] + +# data_type='s3', query=['cohort.csv'] +data = [{'cohort.csv': b'...file content as bytes...'}] +``` + +With `query=None`, an S3 data store returns **all** of its files under their names, while a FHIR data store returns +nothing, because FHIR always needs a query. + +### With the Core SDK + +```python +from flame import FlameCoreSDK + +flame = FlameCoreSDK() + +if flame.node_has_data(): + # list of {query: result} + fhir = flame.get_fhir_data(['Patient?_summary=count']) + + # list of {key: bytes} + files = flame.get_s3_data(['cohort.csv']) + + # every file of every S3 data store + all_files = flame.get_s3_data() +``` + +Both methods return the same structure as `data` above. See +[Data Source Client](/guide/user/sdk-core-doc#data-source-client) for `get_data_sources()` and `get_data_client()`. + +::: warning Aggregators have no data by default +The data methods return `None` on a node without a data connection. This is the case for the aggregator unless the +SDK is constructed with `aggregator_requires_data=True`, and for proxy nodes. Check with `node_has_data()`. +::: + +### Reading FHIR data + +The value for each query is the parsed FHIR response as a dictionary: + +```python +def analysis_method(self, data, aggregator_results): + patient_count = data[0]['Patient?_summary=count']['total'] + return patient_count +``` + +To turn `Observation` or `QuestionnaireResponse` resources into a table, use +[`fhir_to_csv()`](/guide/user/sdk-core-doc#fhir-to-csv). The [FHIR Queries](/guide/user/fhir-query) page explains how +to write queries. + +### Reading files (CSV, VCF, text, ...) + +S3 data arrives as **bytes in memory**, whatever the file format. How you read it depends on the library: + +```python +from io import BytesIO +import tempfile + +import pandas as pd + + +def analysis_method(self, data, aggregator_results): + files = data[0] + + # CSV or other tabular formats: wrap the bytes in a file-like object + df = pd.read_csv(BytesIO(files['cohort.csv'])) + + # Plain text + notes = files['notes.txt'].decode('utf-8') + + # Libraries or command line tools that need a file path + # (e.g. pysam for VCF): write the bytes to a temporary file first + with tempfile.NamedTemporaryFile(mode='wb') as tmp: + tmp.write(files['sample.vcf.gz']) + tmp.flush() + # open tmp.name with your library + ... +``` + +If file names differ between nodes, do not hard-code them. Request all files and select by suffix: + +```python +csv_files = { + name: content + for name, content in data[0].items() + if name.endswith('.csv') +} +``` + +Complete examples: [CSV](/guide/user/coding_examples/federated-logistic-regression), +[VCF](/guide/user/coding_examples/vcf-qc), [FASTQ with a command line tool](/guide/user/coding_examples/cli-fastqc). + +## 2. How do I compute on the node? + +Everything in `analysis_method()` runs **inside the node**, next to the data. Only its return value leaves the node. + +```python +class MyAnalyzer(StarAnalyzer): + def analysis_method(self, data, aggregator_results): + df = pd.read_csv(BytesIO(data[0]['cohort.csv'])) + return {'n': len(df), 'sum_age': float(df['age'].sum())} + + +class MyAggregator(StarAggregator): + def aggregation_method(self, analysis_results): + n = sum(r['n'] for r in analysis_results) + return sum(r['sum_age'] for r in analysis_results) / n + + def has_converged(self, result, last_result): + return True +``` + +Things to keep in mind: + +* **Return aggregates, not rows.** The return value is sent to the aggregator, so it must not contain patient-level + or otherwise identifying data. This includes file names and error messages. +* **Libraries come from the master image.** You cannot install packages at runtime. If the library you need is + missing, see [Requesting a Master Image](/guide/user/master-images). +* **Command line tools** contained in the master image can be called with `subprocess`, see the + [FastQC example](/guide/user/coding_examples/cli-fastqc). +* **Parameters** for your classes are passed through `analyzer_kwargs` and `aggregator_kwargs` on the model. +* **Logging and progress** are available as `self.flame.flame_log()` and `self.flame.set_progress()`. +* **Exceptions** set the analysis to `failed`. The Hub only shows a generic message, the stack trace stays in the + node's local logs (see [Error Handling](/guide/user/analysis-coding#error-handling)). + +### Computing over several rounds + +With `simple_analysis=False` the aggregated result is sent back to the analyzers and `analysis_method()` is called +again, until `has_converged()` returns `True`. The previous aggregate arrives as `aggregator_results`, which is `None` +in the first round: + +```python +def analysis_method(self, data, aggregator_results): + if aggregator_results is not None: + # continue from the global model + self.model.coef_ = aggregator_results + self.model.fit(self.X, self.y) + return self.model.coef_ +``` + +State you set on `self` survives between rounds. For long runs, see +[Checkpointing](/guide/user/analysis-coding#checkpointing). + +### Test before you submit + +Run the same classes on your own machine with `StarModelTester` before uploading the code to the Hub. It prints full +stack traces, which the Hub does not. See [Local Testing](/guide/user/local-testing). + +## 3. How do nodes communicate? + +Nodes never connect to each other directly. All traffic passes through the Hub, using two channels: + +| Channel | For | Limit | Methods | +|---|---|---|---| +| Message Broker | control messages and small values | about 2 MB per message | `send_message()`, `await_messages()` | +| Storage Service | larger data such as model weights | none stated | `send_intermediate_data()`, `await_intermediate_data()` | + +Data sent with `send_intermediate_data()` is always encrypted for the receiving nodes. + +### With a pattern + +You do not call any of these methods. The pattern moves the return values for you: + +* `StarModel`: `analysis_method()` on each analyzer → `aggregation_method()` on the aggregator → back to the analyzers + as `aggregator_results` in the next round. +* `ProxyModel`: `analysis_method()` → `proxy_aggregation_method()` on the assigned proxy → `aggregation_method()` on + the aggregator. The aggregator never sees the result of a single analyzer. + +### With the Core SDK + +First find out who takes part: + +```python +# this node +flame.get_id() + +# 'default', 'aggregator' or 'proxy' +flame.get_role() + +# all other nodes of the analysis +flame.get_participant_ids() + +# the aggregator (None on the aggregator itself) +flame.get_aggregator_id() + +# {node_id: role} of all partners +flame.full_role_call() +``` + +Then exchange messages. A message has a `message_category`, which the receiver uses to pick the messages it waits +for, and a dictionary as content, available on the received message as `.body`: + +```python +from flame import FlameCoreSDK + + +def main(): + flame = FlameCoreSDK() + + if flame.get_role() == 'aggregator': + analyzers = flame.get_participant_ids() + responses = flame.await_messages(senders=analyzers, + message_category='patient_count', + timeout=300) + # a node that did not answer in time has None + # instead of a list of messages + total = sum(msgs[-1].body['count'] + for msgs in responses.values() if msgs) + flame.submit_final_result(result=total, output_type='str') + flame.analysis_finished() + else: + aggregator_id = flame.get_aggregator_id() + query = 'Patient?_summary=count' + count = flame.get_fhir_data([query])[0][query]['total'] + flame.ready_check([aggregator_id]) + flame.send_message(receivers=[aggregator_id], + message_category='patient_count', + message={'count': count}) + + +if __name__ == "__main__": + main() +``` + +For anything larger than a few values, send the data itself through the Storage Service. The two calls combine +storing, messaging and retrieving: + +```python +# sender +flame.send_intermediate_data(receivers=[aggregator_id], + data=model_weights) + +# receiver, returns {node_id: data} +weights = flame.await_intermediate_data(senders=analyzers, + timeout=600) +``` + +Useful helpers: + +* `send_message_and_wait_for_responses()` sends a message and blocks until every receiver has answered in the same + category. +* `ready_check()` waits until partner nodes are up, which avoids sending messages to a node that has not started yet. +* `get_node_index()` gives every node the same deterministic ordering, useful to assign tasks without negotiating. +* `set_role()` with `partner_role_call()` lets you define your own roles beyond analyzer and aggregator. +* `analysis_finished()` tells all partner nodes that the analysis is done. + +All of them are described under [Message Broker Client](/guide/user/sdk-core-doc#message-broker-client), +[Storage Client](/guide/user/sdk-core-doc#storage-client) and [General](/guide/user/sdk-core-doc#general). + +## 4. How do I get results? + +There are two kinds of results: the **final result**, which leaves the federation and is downloaded by the analyst, +and **intermediate results**, which stay inside the analysis or the project. + +| Result | Stored | Readable by | Written by | +|---|---|---|---| +| Final | on the Hub | the analyst | the aggregator only | +| Intermediate, global | on the Hub, encrypted | the nodes it is addressed to, within the same analysis | any node | +| Intermediate, local | on the node | the node itself, also in later analyses of the same project | any node | + +Final results are written with `submit_final_result()`, intermediate results with `save_intermediate_data()` and +`location='global'` or `location='local'`. + +### How aggregation works + +Aggregation is **your code**. FLAME does not average or merge anything on its own, it only delivers the results of +the analyzers to the aggregator. + +1. Every analyzer runs `analysis_method()` and returns its result. +2. The aggregator receives all of them as the list `analysis_results` and runs `aggregation_method()`. +3. With `simple_analysis=True` the return value is submitted as the final result and the analysis ends. +4. With `simple_analysis=False` the aggregator calls `has_converged(result, last_result)` from the second round on. + If it returns `False`, the aggregate is sent back to the analyzers and the next round starts. If it returns + `True`, the aggregate is submitted as the final result. + +`analysis_results` is a plain list. It does not tell you which node sent which entry, so return everything the +aggregation needs as part of the result, for example the sample size for a weighted mean: + +```python +class MyAnalyzer(StarAnalyzer): + def analysis_method(self, data, aggregator_results): + df = pd.read_csv(BytesIO(data[0]['cohort.csv'])) + return {'n': len(df), 'mean_age': float(df['age'].mean())} + + +class MyAggregator(StarAggregator): + def aggregation_method(self, analysis_results): + n = sum(r['n'] for r in analysis_results) + weighted = sum(r['n'] * r['mean_age'] for r in analysis_results) + return weighted / n + + def has_converged(self, result, last_result): + # stop when the result no longer changes, or after 10 rounds + if self.num_iterations >= 10: + return True + return abs(result - last_result) < 1e-6 +``` + +With `ProxyModel` there is one more stage: each proxy pre-aggregates its analyzers in `proxy_aggregation_method()`, +and the aggregator only receives the proxy results. See [Proxy Pattern](/guide/user/proxy-analysis-coding). + +### Final results + +A pattern submits the return value of `aggregation_method()` for you. The format is set on the model: + +```python +StarModel( + ... + # 'str' (text file), 'bytes' (binary file) or 'pickle' + output_type='str', + # optional name of the result file on the Hub + filename='mean_age.txt', +) +``` + +| `output_type` | Use for | Reading the downloaded file | +|---|---|---| +| `'str'` | numbers, text, JSON strings | any text editor | +| `'bytes'` | content you serialized yourself, e.g. an image or a model file | depends on your format | +| `'pickle'` | arbitrary Python objects | `pickle.load()` in an environment with the same libraries | + +To get several files, return a list or tuple and set `multiple_results=True`. `output_type` and `filename` may then +be lists with one entry per element: + +```python +class MyAggregator(StarAggregator): + def aggregation_method(self, analysis_results): + ... + return [json.dumps(summary), model_bytes] + + +StarModel( + ... + multiple_results=True, + output_type=['str', 'bytes'], + filename=['summary.json', 'model.bin'], +) +``` + +With the Core SDK you call the same functionality directly. It only works on the aggregator: + +```python +if flame.get_role() == 'aggregator': + flame.submit_final_result(result=total, output_type='str') +``` + +**Downloading:** once all nodes have finished, open the analysis in the Hub and click **Download** in the +**Results** tab, see [Starting an Analysis](/guide/user/analysis#download-results). + +::: warning Final results leave the federation +Everything you submit is stored on the Hub and can be downloaded. Submit aggregates only. For single numeric results, +noise can be added with `StarLocalDPModel` or the `local_dp` parameter, see +[Local Differential Privacy](/guide/user/analysis-coding#utilizing-local-differential-privacy-in-starmodel). +::: + +### Intermediate and temporary results + +**During one run** you often do not need storage at all: + +* Attributes you set on `self` in your analyzer or aggregator are kept between rounds. +* Files you write to the working directory of the container exist until the analysis ends. They are not part of the + result and are not visible to other nodes. + +**Sharing with other nodes (global):** the data is stored on the Hub, encrypted for exactly the nodes you name, and +can be read by them within the same analysis. This is the channel for anything too large for a message. + +```python +# returns {node_id: {'status': ..., 'url': ..., 'id': ...}} +response = flame.save_intermediate_data(data=model_weights, + location='global', + remote_node_ids=[partner_id]) + +# on the receiving node, with the storage id it was sent +weights = flame.get_intermediate_data(location='global', + query=storage_id) +``` + +The receiver needs the storage `id`. `send_intermediate_data()` and `await_intermediate_data()` store the data and +pass the id along in one step, see [How do nodes communicate?](#_3-how-do-nodes-communicate). + +**Keeping on the node (local):** the data never leaves the node. It outlives the analysis and can be read again by +the same node in a later analysis of the **same project**, which makes it suitable for caching expensive +preprocessing or continuing a training run. Give it a `tag` to find it again: + +```python +# first analysis +flame.save_intermediate_data(data=features, + location='local', + tag='features-v1') + +# later analysis of the same project, on the same node +if 'features-v1' in flame.get_local_tags(): + features = flame.get_intermediate_data(location='local', + tag='features-v1', + tag_option='last') +``` + +Tags consist of lowercase letters, digits and hyphens. Tags containing `checkpoint-` are reserved. + +**Checkpoints** build on local storage and additionally save the files your analysis has written. Use them to resume +a long run, see [Checkpointing](/guide/user/analysis-coding#checkpointing). + +All methods are described under [Storage Client](/guide/user/sdk-core-doc#storage-client).