From 5c8965872146fddbf29ee733a63cbc47fd7f29ba Mon Sep 17 00:00:00 2001 From: Sarah Asad Date: Sun, 3 May 2026 23:13:10 -0700 Subject: [PATCH 01/24] Files added --- .../pythonvirtualenvironment/PveManager.scala | 132 ++++++++++++++-- .../PveResource.scala | 9 +- .../PveWebsocketResource.scala | 36 ++++- .../computing-unit-selection.component.html | 84 +++++++++- .../computing-unit-selection.component.ts | 148 ++++++++++++------ .../virtual-environment.service.ts | 9 +- frontend/src/styles.scss | 10 +- 7 files changed, 352 insertions(+), 76 deletions(-) diff --git a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala index 0399e386ba7..3231565e0ff 100644 --- a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala +++ b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala @@ -32,7 +32,8 @@ import org.apache.texera.amber.config.PythonUtils * for each Computing Unit * * It supports: - * - Creating and initializing isolated Python environments + * - Creating and initializing isolated Python environments (with system packages) + * - installing user defined packages * - Streaming pip output logs back to the caller * * Each PVE is stored under: @@ -41,6 +42,11 @@ import org.apache.texera.amber.config.PythonUtils object PveManager { + case class PvePackageResponse( + pveName: String, + userPackages: Seq[String] + ) + private val VenvRoot: Path = Paths.get("/tmp/texera-pve/venvs") private def cuidDir(cuid: Int, pveName: String): Path = { @@ -121,18 +127,6 @@ object PveManager { return } - if (!Files.exists(requirementsPath)) { - queue.put(s"[PVE][ERR] requirements.txt not found at ${requirementsPath.toAbsolutePath}") - return - } - - if (!Files.exists(operatorRequirementsPath)) { - queue.put( - s"[PVE][ERR] operator-requirements.txt not found at ${operatorRequirementsPath.toAbsolutePath}" - ) - return - } - queue.put( s"[PVE] Installing requirements from ${requirementsPath.toAbsolutePath} and ${operatorRequirementsPath.toAbsolutePath}" ) @@ -170,7 +164,8 @@ object PveManager { queue.put(s"[PVE] Created new environment for cuid = $cuid") } - def getEnvironments(cuid: Int): List[String] = { + // returns list of PVE names and corresponding user packages for a given CU + def getEnvironments(cuid: Int): List[PvePackageResponse] = { val cuPath = VenvRoot.resolve(cuid.toString) @@ -185,7 +180,27 @@ object PveManager { .iterator() .asScala .filter(path => Files.isDirectory(path)) - .map(path => path.getFileName.toString) + .map { path => + val pveName = path.getFileName.toString + val metadataPath = path.resolve("user-packages.txt") + + val userPackages = + if (Files.exists(metadataPath)) { + Files + .readAllLines(metadataPath) + .asScala + .map(_.trim) + .filter(_.nonEmpty) + .toSeq + } else { + Seq() + } + + PvePackageResponse( + pveName = pveName, + userPackages = userPackages + ) + } .toList } finally { stream.close() @@ -212,4 +227,91 @@ object PveManager { stream.close() } } + + /** + * Installs user requested Python packages into the PVE. + * + * 1. Executes pip install for each package + * 2. Updates user metadata file + * 3. Streams logs back via queue + */ + def installUserPackages( + packages: List[String], + cuid: Int, + queue: BlockingQueue[String], + pveName: String + ): Unit = { + + val python = pythonBinPath(cuid, pveName).toAbsolutePath.toString + val envVars = pipEnv + + if (!Files.exists(Paths.get(python))) { + queue.put(s"[PVE][ERR] Python executable not found for PVE: $python") + return + } + + val metadataPath = cuidDir(cuid, pveName).resolve("user-packages.txt") + Files.createDirectories(metadataPath.getParent) + + var installedPackages = + if (Files.exists(metadataPath)) { + Files + .readAllLines(metadataPath) + .asScala + .map(_.trim) + .filter(_.nonEmpty) + .toSet + } else { + Set[String]() + } + + packages.foreach { pkg => + val trimmedPkg = pkg.trim + + if (trimmedPkg.nonEmpty) { + queue.put(s"[PVE] Installing package: $trimmedPkg") + + val code = Process( + Seq( + python, + "-u", + "-m", + "pip", + "install", + "--progress-bar", + "off", + "--no-input", + trimmedPkg + ), + None, + envVars.toSeq: _* + ).!( + ProcessLogger( + out => queue.put(s"[pip] $out"), + err => queue.put(s"[pip][ERR] $err") + ) + ) + + queue.put(s"[pip] install($trimmedPkg) finished with exit code $code") + + if (code != 0) { + queue.put(s"[PVE][ERR] Failed to install package: $trimmedPkg") + return + } + + installedPackages = installedPackages + trimmedPkg + + Files.write( + metadataPath, + installedPackages.toSeq.sorted.asJava + ) + } + } + + queue.put("[PVE] Final user package list:") + + installedPackages.toSeq.sorted.foreach { pkg => + queue.put(s"[user-package] $pkg") + } + } } diff --git a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveResource.scala b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveResource.scala index 1040fd64ea4..0a058ed6f5c 100644 --- a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveResource.scala +++ b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveResource.scala @@ -28,7 +28,7 @@ import java.util @Consumes(Array(MediaType.APPLICATION_JSON)) class PveResource { // -------------------------------------------------- - // Get installed packages + // Get system packages // -------------------------------------------------- @GET @Path("/system") @@ -45,7 +45,7 @@ class PveResource { } // -------------------------------------------------- - // Fetch PVEs + // Fetch PVEs and Installed User Packages // -------------------------------------------------- @GET @Path("/pves") @@ -54,9 +54,10 @@ class PveResource { try { PveManager .getEnvironments(cuid) - .map { pveName => + .map { pve => Map( - "pveName" -> pveName.asInstanceOf[Object] + "pveName" -> pve.pveName.asInstanceOf[Object], + "userPackages" -> pve.userPackages.asJava.asInstanceOf[Object] ).asJava } .asJava diff --git a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveWebsocketResource.scala b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveWebsocketResource.scala index b93d1bfde03..48931bf37c3 100644 --- a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveWebsocketResource.scala +++ b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveWebsocketResource.scala @@ -26,9 +26,9 @@ import scala.concurrent.Future import scala.concurrent.ExecutionContext.Implicits.global /** - * WebSocket endpoint for PVE creation that streams pip installation logs - * to the frontend in real time. The environment setup runs asynchronously, - * and output is pushed to the client until completion. + * WebSocket endpoint for PVE creation and user pacakge installation that streams + * pip installation logs to the frontend in real time. The environment setup runs + * asynchronously, and output is pushed to the client until completion. */ @ServerEndpoint("/wsapi/pve") @@ -42,12 +42,33 @@ class PveWebsocketResource { val cuid = params.get("cuid").get(0).toInt val pveName = params.get("pveName").get(0) val isLocal = params.get("isLocal").get(0).toBoolean + val action = params.getOrDefault("action", java.util.List.of("create")).get(0) val queue = new LinkedBlockingQueue[String]() Future { try { - PveManager.createNewPve(cuid, queue, pveName, isLocal) + action match { + case "create" => + PveManager.createNewPve(cuid, queue, pveName, isLocal) + + case "install" => + val packages = + params + .getOrDefault("packages", java.util.List.of("[]")) + .get(0) + .stripPrefix("[") + .stripSuffix("]") + .split(",") + .toList + .map(_.replace("\"", "").trim) + .filter(_.nonEmpty) + + PveManager.installUserPackages(packages, cuid, queue, pveName) + + case _ => + queue.put(s"[ERR] Unknown action: $action") + } } catch { case e: Exception => queue.put(s"[ERR] ${e.getMessage}") @@ -60,11 +81,10 @@ class PveWebsocketResource { var done = false while (!done && session.isOpen) { - val line = queue.take() - - session.getBasicRemote.sendText(line) + val msg = queue.take() + session.getBasicRemote.sendText(msg) - if (line == "__DONE__") { + if (msg == "__DONE__") { done = true session.close() } diff --git a/frontend/src/app/workspace/component/power-button/computing-unit-selection.component.html b/frontend/src/app/workspace/component/power-button/computing-unit-selection.component.html index b742c71581c..1a0dc7ef339 100644 --- a/frontend/src/app/workspace/component/power-button/computing-unit-selection.component.html +++ b/frontend/src/app/workspace/component/power-button/computing-unit-selection.component.html @@ -480,7 +480,7 @@
-
+
+ + +
+
+
+ +
+ +
+ +
+ +
+ +
+
+
+ + +
+
+
+ + +
+ +
+ + + + + + +
+ +
+ + +
+
+
+ +
+ +
+
-
-
-
- - -
+
+
+
Package
+ +
Version
+
-
- - - - - - -
+
+
+
+ +
-
- - +
+ + + + + +
+ +
+ +
diff --git a/frontend/src/styles.scss b/frontend/src/styles.scss index 7609fa8b955..27b2a5362a2 100644 --- a/frontend/src/styles.scss +++ b/frontend/src/styles.scss @@ -369,6 +369,24 @@ hr { background: transparent; } + .user-package-header-row .package-column-label { + font-weight: 600; + } + + .new-packages-section { + display: flex; + flex-direction: column; + gap: 2px; + } + + .user-package-header-row { + display: grid; + grid-template-columns: 1fr 160px 1fr; + gap: 14px; + margin-bottom: 0; + padding: 0; + } + .system-header { display: flex; flex-direction: column; From b1b16395ecd23aa403788a419a3db06cb6594e0b Mon Sep 17 00:00:00 2001 From: Sarah Asad Date: Fri, 8 May 2026 08:43:28 -0700 Subject: [PATCH 16/24] require op and version --- .../computing-unit-selection.component.ts | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/frontend/src/app/workspace/component/power-button/computing-unit-selection.component.ts b/frontend/src/app/workspace/component/power-button/computing-unit-selection.component.ts index 6cd9a37e29b..969711863ff 100644 --- a/frontend/src/app/workspace/component/power-button/computing-unit-selection.component.ts +++ b/frontend/src/app/workspace/component/power-button/computing-unit-selection.component.ts @@ -974,6 +974,15 @@ export class ComputingUnitSelectionComponent implements OnInit { private installUserPackages(index: number): void { const env = this.pves[index]; + const missingVersionPackage = env.newPackages?.find( + pkg => pkg.name?.trim() && (!pkg.versionOp?.trim() || !pkg.version?.trim()) + ); + + if (missingVersionPackage) { + this.notificationService.error("Please specify an operator and version for each package."); + return; + } + const systemPackageNames = new Set(this.systemPackages.map(pkg => pkg.name.trim().toLowerCase())); const userPackageNames = new Set(env.userPackages.map(pkg => pkg.name.trim().toLowerCase())); @@ -987,18 +996,18 @@ export class ComputingUnitSelectionComponent implements OnInit { const packageName = pkg.name.trim().toLowerCase(); if (systemPackageNames.has(packageName)) { - this.notificationService.error(`Skipped ${pkg.name}: already installed as a system package.`) + this.notificationService.error(`Skipped ${pkg.name}: already installed as a system package.`); return false; } if (userPackageNames.has(packageName)) { - this.notificationService.error(`Skipped ${pkg.name}: already installed in this environment.`) + this.notificationService.error(`Skipped ${pkg.name}: already installed in this environment.`); return false; } return true; }) - .map(pkg => `${pkg.name.trim()}${pkg.version ? `==${pkg.version.trim()}` : ""}`) ?? []; + .map(pkg => `${pkg.name.trim()}${pkg.versionOp}${(pkg.version ?? "").trim()}`) ?? []; if (skippedMessages.length > 0) { this.pves[index].pipOutput = `${this.pves[index].pipOutput ?? ""}` + skippedMessages.join("\n") + "\n"; From a71c4d4277c1a6a86933f865eb3d007df1aee3f9 Mon Sep 17 00:00:00 2001 From: Sarah Asad Date: Fri, 8 May 2026 12:48:12 -0700 Subject: [PATCH 17/24] update comment --- .../web/resource/pythonvirtualenvironment/PveManager.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala index 41c728c4591..140cdbd1daf 100644 --- a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala +++ b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala @@ -117,7 +117,7 @@ object PveManager { queue.put(s"[PVE] Creating new PVE for cuid: $cuid with name: $pveName") // NOTE: These paths are derived from computing-unit-master.dockerfile. - // If requirements.txt or operator-requirements.txt locations change, update these paths. + // If requirements.txt location changes, update these paths. val requirementsPath = if (isLocal) Paths.get("amber", "requirements.txt") else Paths.get("/tmp", "requirements.txt") From a09b091739c6d3d64d55fe3440a881488f689c6a Mon Sep 17 00:00:00 2001 From: Sarah Asad Date: Fri, 8 May 2026 13:25:11 -0700 Subject: [PATCH 18/24] Added back op reqs --- .../pythonvirtualenvironment/PveManager.scala | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala index 140cdbd1daf..22ca792a1fe 100644 --- a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala +++ b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala @@ -117,12 +117,16 @@ object PveManager { queue.put(s"[PVE] Creating new PVE for cuid: $cuid with name: $pveName") // NOTE: These paths are derived from computing-unit-master.dockerfile. - // If requirements.txt location changes, update these paths. + // If requirements.txt or operator-requirements.txt locations change, update these paths. val requirementsPath = if (isLocal) Paths.get("amber", "requirements.txt") else Paths.get("/tmp", "requirements.txt") - if (!Files.exists(requirementsPath)) { + val operatorRequirementsPath = + if (isLocal) Paths.get("amber", "operator-requirements.txt") + else Paths.get("/tmp", "operator-requirements.txt") + + if (!Files.exists(requirementsPath) || !Files.exists(operatorRequirementsPath)) { queue.put(s"[PVE][ERR] System requirements not found") return } @@ -156,7 +160,9 @@ object PveManager { python, Seq( "-r", - requirementsPath.toString + requirementsPath.toString, + "-r", + operatorRequirementsPath.toString ), queue ) From ef49f595e8b8fe7eadf900f6e105eb16257073b0 Mon Sep 17 00:00:00 2001 From: Sarah Asad Date: Fri, 8 May 2026 13:27:34 -0700 Subject: [PATCH 19/24] Added back op reqs --- .../web/resource/pythonvirtualenvironment/PveManager.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala index 22ca792a1fe..bf88fa6112c 100644 --- a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala +++ b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala @@ -153,7 +153,7 @@ object PveManager { } queue.put( - s"[PVE] Installing requirements from ${requirementsPath.toAbsolutePath}" + s"[PVE] Installing requirements from ${requirementsPath.toAbsolutePath} and operator requirements from ${operatorRequirementsPath.toAbsolutePath}" ) val installReqCode = runPipInstall( From 6ea9ca2c5b784ab0a426e2d03ddb9b0a56bb9161 Mon Sep 17 00:00:00 2001 From: Sarah Asad Date: Mon, 11 May 2026 15:57:11 -0700 Subject: [PATCH 20/24] protect system packages --- .../pythonvirtualenvironment/PveManager.scala | 66 ++++++++- .../PveResource.scala | 11 +- .../PveWebsocketResource.scala | 2 +- .../PveResourceSpec.scala | 3 +- amber/system-requirements-lock.txt | 134 ++++++++++++++++++ .../computing-unit-selection.component.ts | 3 +- .../virtual-environment.service.ts | 2 +- 7 files changed, 209 insertions(+), 12 deletions(-) create mode 100644 amber/system-requirements-lock.txt diff --git a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala index bf88fa6112c..8d961f1520d 100644 --- a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala +++ b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala @@ -67,9 +67,25 @@ object PveManager { "PIP_NO_INPUT" -> "1" ) - def getSystemPackages(): Seq[String] = { - val python = PythonUtils.getPythonExecutable - Process(Seq(python, "-m", "pip", "freeze")).!!.split("\n").map(_.trim).filter(_.nonEmpty).toSeq + private def getSystemPath(isLocal: Boolean): Path = { + if (isLocal) { + Paths.get("amber", "system-requirements-lock.txt") + } else { + Paths.get("/tmp", "system-requirements-lock.txt") + } + } + + def getSystemPackages(isLocal: Boolean): Seq[String] = { + if (!Files.exists(getSystemPath(isLocal))) { + Seq() + } else { + Files + .readAllLines(getSystemPath(isLocal)) + .asScala + .map(_.trim) + .filter(line => line.nonEmpty && !line.startsWith("#")) + .toSeq + } } private def runPipInstall( @@ -174,6 +190,17 @@ object PveManager { return } + val freezeCode = Process( + Seq(python, "-m", "pip", "freeze") + ).#>(getSystemPath(isLocal).toFile).! + + queue.put(s"[PVE] system requirements lockfile generated with exit code $freezeCode") + + if (freezeCode != 0) { + queue.put(s"[PVE][ERR] Failed to generate system requirements lockfile") + return + } + queue.put(s"[PVE] Created new environment for cuid = $cuid") } @@ -252,11 +279,11 @@ object PveManager { packages: List[String], cuid: Int, queue: BlockingQueue[String], - pveName: String + pveName: String, + isLocal: Boolean ): Unit = { val python = pythonBinPath(cuid, pveName).toAbsolutePath.toString - val envVars = pipEnv if (!Files.exists(Paths.get(python))) { queue.put(s"[PVE][ERR] Python executable not found for PVE: $python") @@ -277,15 +304,42 @@ object PveManager { Set[String]() } + val systemPackages = + if (Files.exists(getSystemPath(isLocal))) { + Files + .readAllLines(getSystemPath(isLocal)) + .asScala + .map(_.trim) + .filter(_.nonEmpty) + .map(line => line.split("==")(0).trim.toLowerCase) + .toSet + } else { + Set[String]() + } + packages.foreach { pkg => val trimmedPkg = pkg.trim if (trimmedPkg.nonEmpty) { + + val userPackageName = trimmedPkg.split("==")(0).trim.toLowerCase + + if (systemPackages.contains(userPackageName)) { + queue.put( + s"[PVE][ERR] $trimmedPkg is a system package and cannot be installed or modified by the user." + ) + return + } + queue.put(s"[PVE] Installing package: $trimmedPkg") val code = runPipInstall( python, - Seq(trimmedPkg), + Seq( + "--constraint", + getSystemPath(isLocal).toString, + trimmedPkg + ), queue ) diff --git a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveResource.scala b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveResource.scala index 0a058ed6f5c..744cd5e1d6a 100644 --- a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveResource.scala +++ b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveResource.scala @@ -35,12 +35,19 @@ class PveResource { @Produces(Array(MediaType.APPLICATION_JSON)) def getSystemPackages: util.Map[String, util.List[String]] = { try { - val systemPkgs = PveManager.getSystemPackages().toList.asJava + + val isLocal = true + + val systemPkgs = + PveManager.getSystemPackages(isLocal).toList.asJava + Map("system" -> systemPkgs).asJava } catch { case e: Exception => e.printStackTrace() - throw new InternalServerErrorException("Failed to get system packages.") + throw new InternalServerErrorException( + "Failed to get system packages." + ) } } diff --git a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveWebsocketResource.scala b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveWebsocketResource.scala index 7b46640626e..4c3ebf790f5 100644 --- a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveWebsocketResource.scala +++ b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveWebsocketResource.scala @@ -64,7 +64,7 @@ class PveWebsocketResource { .map(_.replace("\"", "").trim) .filter(_.nonEmpty) - PveManager.installUserPackages(packages, cuid, queue, pveName) + PveManager.installUserPackages(packages, cuid, queue, pveName, isLocal) case _ => queue.put(s"[ERR] Unknown action: $action") diff --git a/amber/src/test/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveResourceSpec.scala b/amber/src/test/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveResourceSpec.scala index fa79ddd86c1..10e952c8bdc 100644 --- a/amber/src/test/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveResourceSpec.scala +++ b/amber/src/test/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveResourceSpec.scala @@ -80,7 +80,8 @@ class PveResourceSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEach List(packageSpec), testCuid, queue, - testPveName + testPveName, + isLocal = true ) val logs = queueText() diff --git a/amber/system-requirements-lock.txt b/amber/system-requirements-lock.txt new file mode 100644 index 00000000000..ad0189a2563 --- /dev/null +++ b/amber/system-requirements-lock.txt @@ -0,0 +1,134 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This file is generated manually to track system packages used in PVEs. +# NOTE: This file must be updated whenever requirements.txt or +# operator-requirements.txt changes. + +aiobotocore==2.25.1 +aiohappyeyeballs==2.6.1 +aiohttp==3.13.5 +aioitertools==0.13.0 +aiosignal==1.4.0 +annotated-types==0.7.0 +appdirs==1.4.4 +asn1crypto==1.5.1 +attrs==26.1.0 +betterproto==2.0.0b7 +bidict==0.22.0 +boto3==1.40.53 +botocore==1.40.53 +cached-property==1.5.2 +cachetools==6.2.6 +certifi==2026.4.22 +charset-normalizer==3.4.7 +click==8.3.3 +contourpy==1.3.3 +cycler==0.12.1 +Deprecated==1.2.14 +filelock==3.29.0 +fonttools==4.62.1 +frozenlist==1.8.0 +fs==2.4.16 +fsspec==2025.9.0 +grpclib==0.4.9 +h2==4.3.0 +hf-xet==1.5.0 +hpack==4.1.0 +huggingface_hub==0.36.2 +hyperframe==6.1.0 +idna==3.14 +ImageIO==2.37.3 +iniconfig==1.1.1 +Jinja2==3.1.6 +jmespath==1.1.0 +joblib==1.5.3 +kiwisolver==1.5.0 +lazy-loader==0.5 +loguru==0.7.0 +markdown-it-py==4.2.0 +MarkupSafe==3.0.3 +matplotlib==3.10.9 +mdurl==0.1.2 +mmh3==5.2.1 +mpmath==1.3.0 +multidict==6.7.1 +networkx==3.6.1 +numpy==2.1.0 +overrides==7.4.0 +packaging==26.2 +pampy==0.3.0 +pandas==2.2.3 +pg8000==1.31.5 +pillow==12.2.0 +plotly==5.24.1 +pluggy==1.6.0 +praw==7.6.1 +prawcore==2.4.0 +propcache==0.5.2 +protobuf==7.34.1 +psutil==5.9.0 +pyarrow==21.0.0 +pybase64==1.3.2 +pydantic==2.13.4 +pydantic_core==2.46.4 +Pygments==2.20.0 +pyiceberg==0.11.1 +Pympler==1.1 +pyparsing==3.3.2 +pyroaring==1.1.0 +pytest==7.4.0 +pytest-reraise==2.1.2 +pytest-timeout==2.2.0 +python-dateutil==2.8.2 +pytz==2026.2 +PyYAML==6.0.3 +readerwriterlock==1.0.9 +regex==2026.5.9 +requests==2.34.0 +rich==14.3.4 +ruff==0.14.7 +s3fs==2025.9.0 +s3transfer==0.14.0 +safetensors==0.7.0 +scikit-image==0.25.2 +scikit-learn==1.5.0 +scipy==1.17.1 +scramp==1.4.8 +setuptools==80.10.2 +six==1.17.0 +SQLAlchemy==2.0.37 +strictyaml==1.7.3 +sympy==1.14.0 +tenacity==8.5.0 +threadpoolctl==3.6.0 +tifffile==2026.5.2 +tokenizers==0.22.2 +torch==2.8.0 +tqdm==4.67.3 +transformers==4.57.3 +typing-inspection==0.4.2 +typing_extensions==4.14.1 +tzdata==2026.2 +tzlocal==2.1 +update-checker==0.18.0 +urllib3==2.7.0 +websocket-client==1.9.0 +wordcloud==1.9.3 +wrapt==1.17.3 +yarl==1.23.0 +zstandard==0.25.0 diff --git a/frontend/src/app/workspace/component/power-button/computing-unit-selection.component.ts b/frontend/src/app/workspace/component/power-button/computing-unit-selection.component.ts index 969711863ff..1b3dfb23226 100644 --- a/frontend/src/app/workspace/component/power-button/computing-unit-selection.component.ts +++ b/frontend/src/app/workspace/component/power-button/computing-unit-selection.component.ts @@ -765,6 +765,7 @@ export class ComputingUnitSelectionComponent implements OnInit { getPVEs(): void { const cuId = this.selectedComputingUnit!.computingUnit.cuid; + const isLocal = this.selectedComputingUnit?.computingUnit.type === "local"; this.workflowPveService .fetchPVEs(cuId) @@ -783,7 +784,7 @@ export class ComputingUnitSelectionComponent implements OnInit { })); this.workflowPveService - .getSystemPackages() + .getSystemPackages(isLocal) .pipe(untilDestroyed(this)) .subscribe({ next: installedResp => { diff --git a/frontend/src/app/workspace/service/virtual-environment/virtual-environment.service.ts b/frontend/src/app/workspace/service/virtual-environment/virtual-environment.service.ts index 05df1747754..9c7c123df1e 100644 --- a/frontend/src/app/workspace/service/virtual-environment/virtual-environment.service.ts +++ b/frontend/src/app/workspace/service/virtual-environment/virtual-environment.service.ts @@ -49,7 +49,7 @@ export class WorkflowPveService { return params; } - getSystemPackages(): Observable { + getSystemPackages(isLocal: boolean): Observable { const params = this.buildBaseParams(); return this.http.get("/pve/system", { params }); } From 309cd48d4e400b588fc5ed1ca10f26be176eb14f Mon Sep 17 00:00:00 2001 From: Sarah Asad Date: Mon, 11 May 2026 16:02:05 -0700 Subject: [PATCH 21/24] minor changes --- .../web/resource/pythonvirtualenvironment/PveManager.scala | 7 ++++--- .../pythonvirtualenvironment/PveWebsocketResource.scala | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala index 8d961f1520d..f5431493ffa 100644 --- a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala +++ b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala @@ -272,8 +272,9 @@ object PveManager { * Installs user requested Python packages into the PVE. * * 1. Executes pip install for each package - * 2. Updates user metadata file - * 3. Streams logs back via queue + * 2. Prevents conflicts with system dependencies. + * 3. Updates user metadata file + * 4. Streams logs back via queue */ def installUserPackages( packages: List[String], @@ -336,7 +337,7 @@ object PveManager { val code = runPipInstall( python, Seq( - "--constraint", + "--constraint", // check against system-requirements-lock getSystemPath(isLocal).toString, trimmedPkg ), diff --git a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveWebsocketResource.scala b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveWebsocketResource.scala index 4c3ebf790f5..e21f91fada3 100644 --- a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveWebsocketResource.scala +++ b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveWebsocketResource.scala @@ -26,7 +26,7 @@ import scala.concurrent.Future import scala.concurrent.ExecutionContext.Implicits.global /** - * WebSocket endpoint for PVE creation and user pacakge installation that streams + * WebSocket endpoint for PVE creation and user package installation that streams * pip installation logs to the frontend in real time. The environment setup runs * asynchronously, and output is pushed to the client until completion. */ From 022e9ef62434dae6b4dea2e3b96bf73305014e97 Mon Sep 17 00:00:00 2001 From: Sarah Asad Date: Mon, 11 May 2026 16:19:14 -0700 Subject: [PATCH 22/24] add header --- amber/system-requirements-lock.txt | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/amber/system-requirements-lock.txt b/amber/system-requirements-lock.txt index ad0189a2563..1717fc9a99f 100644 --- a/amber/system-requirements-lock.txt +++ b/amber/system-requirements-lock.txt @@ -1,24 +1,3 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# This file is generated manually to track system packages used in PVEs. -# NOTE: This file must be updated whenever requirements.txt or -# operator-requirements.txt changes. - aiobotocore==2.25.1 aiohappyeyeballs==2.6.1 aiohttp==3.13.5 From 0a26f6d54d525e74cfae143e3cba618438284837 Mon Sep 17 00:00:00 2001 From: Sarah Asad Date: Mon, 11 May 2026 18:48:05 -0700 Subject: [PATCH 23/24] minor changes --- .../pythonvirtualenvironment/PveManager.scala | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala index f5431493ffa..66599d0a89e 100644 --- a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala +++ b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala @@ -190,17 +190,6 @@ object PveManager { return } - val freezeCode = Process( - Seq(python, "-m", "pip", "freeze") - ).#>(getSystemPath(isLocal).toFile).! - - queue.put(s"[PVE] system requirements lockfile generated with exit code $freezeCode") - - if (freezeCode != 0) { - queue.put(s"[PVE][ERR] Failed to generate system requirements lockfile") - return - } - queue.put(s"[PVE] Created new environment for cuid = $cuid") } From 6b1a1d5d4aaf4c7f6246c4bb7ba24b18da74fe64 Mon Sep 17 00:00:00 2001 From: Sarah Asad Date: Tue, 12 May 2026 05:10:30 -0700 Subject: [PATCH 24/24] minor changes --- .../pythonvirtualenvironment/PveManager.scala | 60 +++++++---------- .../PveResource.scala | 1 + amber/system-requirements-lock.txt | 67 ++++++++----------- 3 files changed, 52 insertions(+), 76 deletions(-) diff --git a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala index 66599d0a89e..27a3b7be7c1 100644 --- a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala +++ b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveManager.scala @@ -67,14 +67,26 @@ object PveManager { "PIP_NO_INPUT" -> "1" ) - private def getSystemPath(isLocal: Boolean): Path = { - if (isLocal) { - Paths.get("amber", "system-requirements-lock.txt") + private def readPackageFile(path: Path): Seq[String] = { + if (Files.exists(path)) { + Files + .readAllLines(path) + .asScala + .map(_.trim) + .filter(_.nonEmpty) + .toSeq } else { - Paths.get("/tmp", "system-requirements-lock.txt") + Seq() } } + private def getSystemPath(isLocal: Boolean): Path = { + Paths.get( + if (isLocal) "amber/system-requirements-lock.txt" + else "/tmp/system-requirements-lock.txt" + ) + } + def getSystemPackages(isLocal: Boolean): Seq[String] = { if (!Files.exists(getSystemPath(isLocal))) { Seq() @@ -133,16 +145,12 @@ object PveManager { queue.put(s"[PVE] Creating new PVE for cuid: $cuid with name: $pveName") // NOTE: These paths are derived from computing-unit-master.dockerfile. - // If requirements.txt or operator-requirements.txt locations change, update these paths. + // If requirements.txt location changes, update these paths. val requirementsPath = if (isLocal) Paths.get("amber", "requirements.txt") else Paths.get("/tmp", "requirements.txt") - val operatorRequirementsPath = - if (isLocal) Paths.get("amber", "operator-requirements.txt") - else Paths.get("/tmp", "operator-requirements.txt") - - if (!Files.exists(requirementsPath) || !Files.exists(operatorRequirementsPath)) { + if (!Files.exists(requirementsPath)) { queue.put(s"[PVE][ERR] System requirements not found") return } @@ -169,16 +177,14 @@ object PveManager { } queue.put( - s"[PVE] Installing requirements from ${requirementsPath.toAbsolutePath} and operator requirements from ${operatorRequirementsPath.toAbsolutePath}" + s"[PVE] Installing requirements from ${requirementsPath.toAbsolutePath}" ) val installReqCode = runPipInstall( python, Seq( "-r", - requirementsPath.toString, - "-r", - operatorRequirementsPath.toString + requirementsPath.toString ), queue ) @@ -213,17 +219,7 @@ object PveManager { val pveName = path.getFileName.toString val metadataPath = path.resolve("user-packages.txt") - val userPackages = - if (Files.exists(metadataPath)) { - Files - .readAllLines(metadataPath) - .asScala - .map(_.trim) - .filter(_.nonEmpty) - .toSeq - } else { - Seq() - } + val userPackages = readPackageFile(metadataPath) PvePackageResponse( pveName = pveName, @@ -282,17 +278,7 @@ object PveManager { val metadataPath = cuidDir(cuid, pveName).resolve("user-packages.txt") - var installedPackages = - if (Files.exists(metadataPath)) { - Files - .readAllLines(metadataPath) - .asScala - .map(_.trim) - .filter(_.nonEmpty) - .toSet - } else { - Set[String]() - } + var installedPackages = readPackageFile(metadataPath).toSet val systemPackages = if (Files.exists(getSystemPath(isLocal))) { @@ -300,7 +286,7 @@ object PveManager { .readAllLines(getSystemPath(isLocal)) .asScala .map(_.trim) - .filter(_.nonEmpty) + .filter(line => line.nonEmpty && !line.startsWith("#")) .map(line => line.split("==")(0).trim.toLowerCase) .toSet } else { diff --git a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveResource.scala b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveResource.scala index 744cd5e1d6a..8a6f4875293 100644 --- a/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveResource.scala +++ b/amber/src/main/scala/org/apache/texera/web/resource/pythonvirtualenvironment/PveResource.scala @@ -36,6 +36,7 @@ class PveResource { def getSystemPackages: util.Map[String, util.List[String]] = { try { + // TODO: Support Kubernetes environment handling val isLocal = true val systemPkgs = diff --git a/amber/system-requirements-lock.txt b/amber/system-requirements-lock.txt index 1717fc9a99f..67ea725c88c 100644 --- a/amber/system-requirements-lock.txt +++ b/amber/system-requirements-lock.txt @@ -1,7 +1,28 @@ -aiobotocore==2.25.1 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This file is manually generated to track system packages used in PVEs. +# NOTE: This file must be updated whenever requirements.txt or +# operator-requirements.txt changes. + aiohappyeyeballs==2.6.1 aiohttp==3.13.5 aioitertools==0.13.0 +aiobotocore==2.25.1 aiosignal==1.4.0 annotated-types==0.7.0 appdirs==1.4.4 @@ -11,50 +32,33 @@ betterproto==2.0.0b7 bidict==0.22.0 boto3==1.40.53 botocore==1.40.53 -cached-property==1.5.2 +cached_property==1.5.2 cachetools==6.2.6 certifi==2026.4.22 -charset-normalizer==3.4.7 +charset_normalizer==3.4.7 click==8.3.3 -contourpy==1.3.3 -cycler==0.12.1 Deprecated==1.2.14 -filelock==3.29.0 -fonttools==4.62.1 frozenlist==1.8.0 fs==2.4.16 fsspec==2025.9.0 grpclib==0.4.9 h2==4.3.0 -hf-xet==1.5.0 hpack==4.1.0 -huggingface_hub==0.36.2 hyperframe==6.1.0 idna==3.14 -ImageIO==2.37.3 iniconfig==1.1.1 -Jinja2==3.1.6 jmespath==1.1.0 -joblib==1.5.3 -kiwisolver==1.5.0 -lazy-loader==0.5 loguru==0.7.0 markdown-it-py==4.2.0 -MarkupSafe==3.0.3 -matplotlib==3.10.9 mdurl==0.1.2 mmh3==5.2.1 -mpmath==1.3.0 multidict==6.7.1 -networkx==3.6.1 numpy==2.1.0 overrides==7.4.0 packaging==26.2 pampy==0.3.0 pandas==2.2.3 pg8000==1.31.5 -pillow==12.2.0 -plotly==5.24.1 pluggy==1.6.0 praw==7.6.1 prawcore==2.4.0 @@ -62,12 +66,11 @@ propcache==0.5.2 protobuf==7.34.1 psutil==5.9.0 pyarrow==21.0.0 -pybase64==1.3.2 pydantic==2.13.4 -pydantic_core==2.46.4 -Pygments==2.20.0 +pydantic-core==2.46.4 +pygments==2.20.0 pyiceberg==0.11.1 -Pympler==1.1 +pympler==1.1 pyparsing==3.3.2 pyroaring==1.1.0 pytest==7.4.0 @@ -75,31 +78,18 @@ pytest-reraise==2.1.2 pytest-timeout==2.2.0 python-dateutil==2.8.2 pytz==2026.2 -PyYAML==6.0.3 readerwriterlock==1.0.9 -regex==2026.5.9 requests==2.34.0 rich==14.3.4 ruff==0.14.7 s3fs==2025.9.0 s3transfer==0.14.0 -safetensors==0.7.0 -scikit-image==0.25.2 -scikit-learn==1.5.0 -scipy==1.17.1 scramp==1.4.8 setuptools==80.10.2 six==1.17.0 SQLAlchemy==2.0.37 strictyaml==1.7.3 -sympy==1.14.0 tenacity==8.5.0 -threadpoolctl==3.6.0 -tifffile==2026.5.2 -tokenizers==0.22.2 -torch==2.8.0 -tqdm==4.67.3 -transformers==4.57.3 typing-inspection==0.4.2 typing_extensions==4.14.1 tzdata==2026.2 @@ -107,7 +97,6 @@ tzlocal==2.1 update-checker==0.18.0 urllib3==2.7.0 websocket-client==1.9.0 -wordcloud==1.9.3 wrapt==1.17.3 yarl==1.23.0 -zstandard==0.25.0 +zstandard==0.25.0 \ No newline at end of file