Skip to content

Commit 306885f

Browse files
Matt-CarreMatt-Carre
committed
docs(copier): includes helm template in copier
1 parent ed04c7a commit 306885f

10 files changed

Lines changed: 472 additions & 1 deletion

File tree

‎src/copier_template/.github/workflows/_update_image.yml‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
name: Update Docker Image
2+
3+
on:
4+
workflow_call:
5+
6+
jobs:
7+
build:
8+
runs-on: ubuntu-latest
9+
permissions:
10+
contents: read
11+
packages: write
12+
steps:
13+
- name: Checkout Code
14+
uses: actions/checkout@v6
15+
16+
- name: Generate Image Name
17+
run: echo IMAGE_REPOSITORY=ghcr.io/$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]' | tr '[_]' '[\-]')-mounted-in-image >> $GITHUB_ENV
18+
19+
- name: Log in to GitHub Docker Registry
20+
if: github.event_name != 'pull_request'
21+
uses: docker/login-action@v4.1.0
22+
with:
23+
registry: ghcr.io
24+
username: ${{ github.actor }}
25+
password: ${{ secrets.GITHUB_TOKEN }}
26+
27+
- name: Docker Metadata
28+
id: meta
29+
uses: docker/metadata-action@v6.1.0
30+
with:
31+
images: ${{ env.IMAGE_REPOSITORY }}
32+
tags: |
33+
type=ref,event=branch
34+
type=raw,value=latest,enable={{is_default_branch}}
35+
type=match,pattern=python-interface-to-workflows@v?(.+),group=1
36+
37+
- name: Set up Docker Buildx
38+
uses: docker/setup-buildx-action@v4.0.0
39+
with:
40+
driver-opts: network=host
41+
42+
- name: Build Image
43+
uses: docker/build-push-action@v6.18.0
44+
with:
45+
context: .
46+
push: ${{ github.event_name == 'push' }}
47+
load: false
48+
tags: ${{ steps.meta.outputs.tags }}
49+
labels: ${{ steps.meta.outputs.labels }}
50+
cache-from: type=gha
51+
cache-to: type=gha,mode=max

‎src/copier_template/src/{{ project_name }}/helm/Chart.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
apiVersion: v2
2+
name: examples
3+
description: A chart example of WorkflowTemplates
4+
type: application
5+
version: "0.1"

‎src/copier_template/src/{{ project_name }}/helm/notebooks/pandas.ipynb‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "markdown",
5+
"metadata": {},
6+
"source": [
7+
"# Data Manipulation and Visualization Example\n",
8+
"\n",
9+
"This notebook demonstrates:\n",
10+
"- Creating synthetic test data\n",
11+
"- Performing data manipulation with pandas\n",
12+
"- Visualizing the results with matplotlib"
13+
]
14+
},
15+
{
16+
"cell_type": "code",
17+
"execution_count": null,
18+
"metadata": {},
19+
"outputs": [],
20+
"source": [
21+
"import os\n",
22+
"\n",
23+
"# Set environment variables\n",
24+
"os.environ[\"MPLCONFIGDIR\"] = \"/tmp/.config/matplotlib\"\n",
25+
"\n",
26+
"# Ensure the directories exist\n",
27+
"os.makedirs(os.environ[\"MPLCONFIGDIR\"], exist_ok=True)\n",
28+
"\n",
29+
"# Import required libraries\n",
30+
"import pandas as pd\n",
31+
"import numpy as np\n",
32+
"import matplotlib.pyplot as plt\n",
33+
"\n",
34+
"# Set a random seed for reproducibility\n",
35+
"np.random.seed(42)"
36+
]
37+
},
38+
{
39+
"cell_type": "code",
40+
"execution_count": null,
41+
"metadata": {},
42+
"outputs": [],
43+
"source": [
44+
"# Create synthetic test data\n",
45+
"dates = pd.date_range(start='2023-01-01', periods=100)\n",
46+
"categories = ['A', 'B', 'C']\n",
47+
"\n",
48+
"data = pd.DataFrame({\n",
49+
" 'Date': dates,\n",
50+
" 'Category': np.random.choice(categories, size=100),\n",
51+
" 'Value': np.random.normal(loc=50, scale=10, size=100)\n",
52+
"})\n",
53+
"\n",
54+
"# Display first few rows\n",
55+
"data.head()"
56+
]
57+
},
58+
{
59+
"cell_type": "code",
60+
"execution_count": null,
61+
"metadata": {},
62+
"outputs": [],
63+
"source": [
64+
"# Data manipulation\n",
65+
"# 1. Add a rolling average column\n",
66+
"data['RollingAvg'] = data['Value'].rolling(window=7).mean()\n",
67+
"\n",
68+
"# 2. Group by Category and calculate mean value\n",
69+
"category_means = data.groupby('Category')['Value'].mean()\n",
70+
"category_means"
71+
]
72+
},
73+
{
74+
"cell_type": "code",
75+
"execution_count": null,
76+
"metadata": {},
77+
"outputs": [],
78+
"source": [
79+
"# Visualization\n",
80+
"plt.figure(figsize=(12, 6))\n",
81+
"\n",
82+
"# Plot the original Value and Rolling Average for each category\n",
83+
"for cat in data['Category'].unique():\n",
84+
" subset = data[data['Category'] == cat]\n",
85+
" plt.plot(subset['Date'], subset['Value'], label=f'{cat} Value', alpha=0.3)\n",
86+
" plt.plot(subset['Date'], subset['RollingAvg'], label=f'{cat} RollingAvg')\n",
87+
"\n",
88+
"plt.xlabel('Date')\n",
89+
"plt.ylabel('Value')\n",
90+
"plt.title('Value and Rolling Average by Category')\n",
91+
"plt.legend()\n",
92+
"plt.grid(True)\n",
93+
"plt.tight_layout()\n",
94+
"plt.show()"
95+
]
96+
}
97+
],
98+
"metadata": {
99+
"kernelspec": {
100+
"display_name": "Python 3",
101+
"language": "python",
102+
"name": "python3"
103+
},
104+
"language_info": {
105+
"name": "python",
106+
"version": ""
107+
}
108+
},
109+
"nbformat": 4,
110+
"nbformat_minor": 2
111+
}

‎src/copier_template/src/{{ project_name }}/helm/notebooks/requirements.txt‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
numpy==2.2.5
2+
matplotlib==3.10.3
3+
pandas==2.2.3
4+
scipy==1.15.3
5+
nbconvert==7.17.1
6+
ipykernel==6.29.5
7+
ipython==9.2.0
8+
papermill==2.6.0

‎src/copier_template/src/{{ project_name }}/helm/templates/notebook.txt.jinja‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
{% raw %}
2+
# This is from workflows/example https://github.com/DiamondLightSource/workflows/blob/main/examples/helm-based-templates/templates/notebook.yaml
3+
apiVersion: argoproj.io/v1alpha1
4+
kind: ClusterWorkflowTemplate
5+
metadata:
6+
name: notebook
7+
labels:
8+
workflows.diamond.ac.uk/science-group-examples: "true"
9+
annotations:
10+
workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/{% endraw %}{{project_name}}{% raw %}"
11+
spec:
12+
entrypoint: notebook
13+
volumeClaimTemplates:
14+
- metadata:
15+
name: tmp
16+
spec:
17+
accessModes:
18+
- ReadWriteOnce
19+
resources:
20+
requests:
21+
storage: 8Gi
22+
storageClassName: netapp
23+
podResources:
24+
requests:
25+
cpu: 600m
26+
memory: 512Mi
27+
limits:
28+
cpu: "1"
29+
memory: 1Gi
30+
podSpecPatch: |
31+
containers:
32+
- name: main
33+
resources:
34+
limits:
35+
memory: "1Gi"
36+
templates:
37+
- name: mount-files
38+
script:
39+
image: docker.io/library/python:bookworm
40+
command: [bash]
41+
source: |
42+
echo '{{ .Files.Get "notebooks/pandas.ipynb" | b64enc }}' | base64 -d > /tmp/notebook.ipynb
43+
echo '{{ .Files.Get "notebooks/requirements.txt" | b64enc }}' | base64 -d > /tmp/requirements.txt
44+
volumeMounts:
45+
- name: tmp
46+
mountPath: /tmp
47+
- name: convert-notebook
48+
podSpecPatch: '{"containers":[{"name":"main", "resources":{"limits":{"cpu": "600m"}}}]}'
49+
tolerations:
50+
- effect: NoSchedule
51+
key: nvidia.com/gpu
52+
operator: Exists
53+
- effect: NoSchedule
54+
key: nodetype
55+
operator: Equal
56+
value: gpu
57+
- effect: NoSchedule
58+
key: nodegroup
59+
operator: Equal
60+
value: workflows
61+
script:
62+
image: docker.io/library/python:bookworm
63+
command: [bash]
64+
source: |
65+
python -m venv /tmp/venv
66+
/tmp/venv/bin/pip install -r /tmp/requirements.txt
67+
/tmp/venv/bin/python -m ipykernel install --prefix=/tmp/venv --name=venv
68+
/tmp/venv/bin/python -m jupyter nbconvert --execute --allow-errors --to html --output notebook --output-dir /tmp /tmp/notebook.ipynb
69+
volumeMounts:
70+
- name: tmp
71+
mountPath: /tmp
72+
outputs:
73+
artifacts:
74+
- name: notebook
75+
path: /tmp/notebook.html
76+
archive:
77+
none: {}
78+
79+
- name: notebook
80+
dag:
81+
tasks:
82+
- name: files
83+
template: mount-files
84+
- name: convert
85+
template: convert-notebook
86+
dependencies: [files]
87+
{% endraw %}

‎src/copier_template/src/{{ project_name }}/workflow_definitions/create_notebook_in_image.py.jinja‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ from hera.workflows.archive import NoneArchiveStrategy
1818
# and was created with the included dockerfile.
1919
global_config.set_class_defaults( # pyright: ignore
2020
Script,
21-
image="ghcr.io/matt-carre/{% endraw %}{{repo_name}}{% raw %}-mounted-image:latest",
21+
image="ghcr.io/matt-carre/python-interface-to-workflows-mounted-image:latest",
2222
)
2323

2424

‎src/python_interface_to_workflows/helm/Chart.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
apiVersion: v2
2+
name: examples
3+
description: A chart example of WorkflowTemplates
4+
type: application
5+
version: "0.1"

‎src/python_interface_to_workflows/helm/notebooks/pandas.ipynb‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "markdown",
5+
"metadata": {},
6+
"source": [
7+
"# Data Manipulation and Visualization Example\n",
8+
"\n",
9+
"This notebook demonstrates:\n",
10+
"- Creating synthetic test data\n",
11+
"- Performing data manipulation with pandas\n",
12+
"- Visualizing the results with matplotlib"
13+
]
14+
},
15+
{
16+
"cell_type": "code",
17+
"execution_count": null,
18+
"metadata": {},
19+
"outputs": [],
20+
"source": [
21+
"import os\n",
22+
"\n",
23+
"# Set environment variables\n",
24+
"os.environ[\"MPLCONFIGDIR\"] = \"/tmp/.config/matplotlib\"\n",
25+
"\n",
26+
"# Ensure the directories exist\n",
27+
"os.makedirs(os.environ[\"MPLCONFIGDIR\"], exist_ok=True)\n",
28+
"\n",
29+
"# Import required libraries\n",
30+
"import pandas as pd\n",
31+
"import numpy as np\n",
32+
"import matplotlib.pyplot as plt\n",
33+
"\n",
34+
"# Set a random seed for reproducibility\n",
35+
"np.random.seed(42)"
36+
]
37+
},
38+
{
39+
"cell_type": "code",
40+
"execution_count": null,
41+
"metadata": {},
42+
"outputs": [],
43+
"source": [
44+
"# Create synthetic test data\n",
45+
"dates = pd.date_range(start='2023-01-01', periods=100)\n",
46+
"categories = ['A', 'B', 'C']\n",
47+
"\n",
48+
"data = pd.DataFrame({\n",
49+
" 'Date': dates,\n",
50+
" 'Category': np.random.choice(categories, size=100),\n",
51+
" 'Value': np.random.normal(loc=50, scale=10, size=100)\n",
52+
"})\n",
53+
"\n",
54+
"# Display first few rows\n",
55+
"data.head()"
56+
]
57+
},
58+
{
59+
"cell_type": "code",
60+
"execution_count": null,
61+
"metadata": {},
62+
"outputs": [],
63+
"source": [
64+
"# Data manipulation\n",
65+
"# 1. Add a rolling average column\n",
66+
"data['RollingAvg'] = data['Value'].rolling(window=7).mean()\n",
67+
"\n",
68+
"# 2. Group by Category and calculate mean value\n",
69+
"category_means = data.groupby('Category')['Value'].mean()\n",
70+
"category_means"
71+
]
72+
},
73+
{
74+
"cell_type": "code",
75+
"execution_count": null,
76+
"metadata": {},
77+
"outputs": [],
78+
"source": [
79+
"# Visualization\n",
80+
"plt.figure(figsize=(12, 6))\n",
81+
"\n",
82+
"# Plot the original Value and Rolling Average for each category\n",
83+
"for cat in data['Category'].unique():\n",
84+
" subset = data[data['Category'] == cat]\n",
85+
" plt.plot(subset['Date'], subset['Value'], label=f'{cat} Value', alpha=0.3)\n",
86+
" plt.plot(subset['Date'], subset['RollingAvg'], label=f'{cat} RollingAvg')\n",
87+
"\n",
88+
"plt.xlabel('Date')\n",
89+
"plt.ylabel('Value')\n",
90+
"plt.title('Value and Rolling Average by Category')\n",
91+
"plt.legend()\n",
92+
"plt.grid(True)\n",
93+
"plt.tight_layout()\n",
94+
"plt.show()"
95+
]
96+
}
97+
],
98+
"metadata": {
99+
"kernelspec": {
100+
"display_name": "Python 3",
101+
"language": "python",
102+
"name": "python3"
103+
},
104+
"language_info": {
105+
"name": "python",
106+
"version": ""
107+
}
108+
},
109+
"nbformat": 4,
110+
"nbformat_minor": 2
111+
}

0 commit comments

Comments
 (0)