[ZEPPELIN-6717] Install locked Python dependencies in Linux CI

.github/workflows/frontend.yml

@@ -180,27 +180,39 @@ jobs:
         with:
           distribution: 'temurin'
           java-version: 11
-      - name: Cache local Maven repository
+      - name: Cache Maven dependencies
         uses: actions/cache@v5
         with:
           path: |
             ~/.m2/repository
             !~/.m2/repository/org/apache/zeppelin/
-            ~/.spark-dist
-            ~/.cache
-          key: ${{ runner.os }}-zeppelin-${{ hashFiles('**/pom.xml') }}
+            ~/.m2/wrapper
+          key: ${{ runner.os }}-${{ runner.arch }}-maven-v2-${{ hashFiles('**/pom.xml', '.mvn/**') }}
           restore-keys: |
-            ${{ runner.os }}-zeppelin-
+            ${{ runner.os }}-${{ runner.arch }}-maven-v2-
+      - name: Cache npm packages
+        uses: actions/cache@v5
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-${{ runner.arch }}-npm-v1-${{ hashFiles('zeppelin-web/package-lock.json', 'zeppelin-web-angular/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ runner.arch }}-npm-v1-
+      - name: Cache locked Conda packages
+        uses: actions/cache@v5
+        with:
+          path: ~/conda_pkgs_dir
+          key: ${{ runner.os }}-${{ runner.arch }}-conda-python-3.9-v1-${{ hashFiles('testing/locks/python-3.9-linux-64.conda.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ runner.arch }}-conda-python-3.9-v1-
       - name: Setup conda environment with python 3.9
         uses: conda-incubator/setup-miniconda@v4
         with:
           activate-environment: python_3
-          environment-file: testing/env_python_3.yml
-          python-version: 3.9
-          channels: conda-forge,defaults
-          channel-priority: strict
+          environment-file: testing/locks/python-3.9-linux-64.conda.lock
+          channels: conda-forge
+          conda-remove-defaults: true
+          auto-update-conda: false
           auto-activate: false
-          use-mamba: true
       - name: Install Environment
         run: |
           ./mvnw clean install -DskipTests -am -pl zeppelin-integration -Pweb-classic -Pintegration -Pspark-scala-2.12 -Pspark-3.5 -Pweb-dist ${MAVEN_ARGS}

testing/env_python_3.9.yml

@@ -1,8 +1,8 @@
 name: python_3
 channels:
   - conda-forge
-  - defaults
 dependencies:
+  - python=3.9
   - pycodestyle
   - scipy
   - numpy=1.19.5

testing/env_python_3.yml

@@ -1,8 +1,8 @@
 name: python_3
 channels:
   - conda-forge
-  - defaults
 dependencies:
+  - python=3.9
   - pycodestyle
   - scipy
   - numpy=1.19.5

testing/env_python_3_with_flink_119.yml

@@ -1,8 +1,8 @@
 name: python_3_with_flink
 channels:
   - conda-forge
-  - defaults
 dependencies:
+  - python=3.9
   - pycodestyle
   - scipy
   - numpy=1.19.5

testing/env_python_3_with_flink_120.yml

@@ -1,8 +1,8 @@
 name: python_3_with_flink
 channels:
   - conda-forge
-  - defaults
 dependencies:
+  - python=3.9
   - pycodestyle
   - scipy
   - numpy=1.19.5

testing/env_python_3_with_tensorflow.yml

@@ -1,8 +1,8 @@
 name: python_3_with_tensorflow
 channels:
   - conda-forge
-  - defaults
 dependencies:
+  - python=3.9
   - pycodestyle
   - scipy
   - numpy=1.19.5

testing/locks/README.md

@@ -0,0 +1,50 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+# Python dependencies in Linux CI
+
+For the Linux CI environments listed below, the YAML files in `testing/` describe the dependencies we want, while CI installs the resolved Python 3.9 packages from the files here. Changing one of those YAML files alone will not change its CI environment; update the corresponding lock as well.
+
+| Lock file | Environment it supplies |
+|---|---|
+| `python-3.9-linux-64.conda.lock` | Shared Conda environment for core, Python/Jupyter, integration, Spark, Livy, Flink, and frontend integration jobs. It covers `env_python_3.9.yml`, `env_python_3.yml`, and the Conda dependencies in both `env_python_3_with_flink_*.yml` files. |
+| `python-3.9-tensorflow-linux-64.conda.lock` | Conda environment for non-core interpreter jobs, described by `env_python_3_with_tensorflow.yml`. |
+| `pyflink-1.19-python-3.9.requirements.lock` | PyPI packages for the Flink 1.19 job, starting from the pip requirement in `env_python_3_with_flink_119.yml`. |
+| `pyflink-1.20-python-3.9.requirements.lock` | PyPI packages for the Flink 1.20 job, starting from the pip requirement in `env_python_3_with_flink_120.yml`. |
+
+## Changing dependencies
+
+1. Edit the relevant YAML file in `testing/`. If a shared Conda dependency changes, check that the other environments using the shared lock still have the packages they need.
+2. On Ubuntu 24.04 x86-64, resolve the changed Conda environment using conda-forge and export it with `conda list --explicit --md5`. For example, to update the shared lock:
+
+   ```bash
+   conda env create -n zeppelin-lock-source -f testing/env_python_3.9.yml
+   conda list -n zeppelin-lock-source --explicit --md5
+   ```
+
+   Put the exported explicit specification in the corresponding `.conda.lock` file, retaining its ASF license header. Use `env_python_3_with_tensorflow.yml` instead when updating the TensorFlow lock.
+3. For a PyFlink change, start with a fresh environment installed from the shared Conda lock. Install the `apache-flink` version specified in the matching Flink YAML file with pip, then record every PyPI package and version it installs in the matching `.requirements.lock` file. CI uses `pip install --no-deps`, so the file must include transitive PyPI dependencies, not just `apache-flink`.
+4. Validate the lock in a new environment, not just the environment used to generate it. For the shared lock and Flink 1.19, for example:
+
+   ```bash
+   conda create -n zeppelin-lock-check --file testing/locks/python-3.9-linux-64.conda.lock
+   conda run -n zeppelin-lock-check python -m pip install --no-deps -r testing/locks/pyflink-1.19-python-3.9.requirements.lock
+   ```
+
+   Use the TensorFlow lock or Flink 1.20 requirements as appropriate, then run the affected imports and CI tests.
+
+The `.conda.lock` files contain exact Linux x86-64 package URLs and MD5 hashes; they are not portable to other platforms. The PyFlink files pin package versions but do not pin wheel hashes. A cache miss may still download packages, but installation from the Conda locks does not run the dependency solver.

testing/locks/pyflink-1.19-python-3.9.requirements.lock

@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apache-beam==2.48.0
+apache-flink==1.19.3
+apache-flink-libraries==1.19.3
+avro-python3==1.10.2
+cloudpickle==2.2.1
+crcmod==1.7
+dill==0.3.1.1
+dnspython==2.7.0
+docopt==0.6.2
+fastavro==1.12.2
+find-libpython==0.5.1
+hdfs==2.7.3
+httplib2==0.22.0
+numpy==1.24.4
+objsize==0.6.1
+orjson==3.11.5
+pemja==0.4.1
+proto-plus==1.27.1
+py4j==0.10.9.7
+pydot==1.4.2
+pymongo==4.18.1
+regex==2026.1.15
+ruamel.yaml==0.19.1

testing/locks/pyflink-1.20-python-3.9.requirements.lock

@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apache-beam==2.48.0
+apache-flink==1.20.4
+apache-flink-libraries==1.20.4
+avro-python3==1.10.2
+cloudpickle==2.2.1
+crcmod==1.7
+dill==0.3.1.1
+dnspython==2.7.0
+docopt==0.6.2
+fastavro==1.12.2
+find-libpython==0.5.1
+hdfs==2.7.3
+httplib2==0.22.0
+numpy==1.24.4
+objsize==0.6.1
+orjson==3.11.5
+pemja==0.5.6
+proto-plus==1.27.1
+py4j==0.10.9.7
+pydot==1.4.2
+pymongo==4.18.1
+regex==2026.1.15
+ruamel.yaml==0.19.1