diff --git a/frontend/src/app/common/service/computing-unit/workflow-computing-unit/workflow-computing-unit-managing.service.spec.ts b/frontend/src/app/common/service/computing-unit/workflow-computing-unit/workflow-computing-unit-managing.service.spec.ts index b7aa8218bc2..021a59d14ac 100644 --- a/frontend/src/app/common/service/computing-unit/workflow-computing-unit/workflow-computing-unit-managing.service.spec.ts +++ b/frontend/src/app/common/service/computing-unit/workflow-computing-unit/workflow-computing-unit-managing.service.spec.ts @@ -29,6 +29,7 @@ import { COMPUTING_UNIT_LIST_URL, COMPUTING_UNIT_TYPES_URL, } from "./workflow-computing-unit-managing.service"; +import { ClusterCapacity } from "../../../type/workflow-computing-unit"; describe("WorkflowComputingUnitManagingService", () => { let service: WorkflowComputingUnitManagingService; @@ -183,6 +184,34 @@ describe("WorkflowComputingUnitManagingService", () => { expect(status).toBe(403); }); + // Spelled out, so a typo in the constant cannot pass by matching itself. + it("getClusterCapacity() GETs the admin cluster capacity endpoint and returns the body as is", () => { + const capacity: ClusterCapacity = { + cpu: { total: 16000, allocated: 12500, available: 3500 }, + memory: { total: 68719476736, allocated: 17179869184, available: 51539607552 }, + gpu: { total: 0, allocated: 0, available: 0 }, + }; + let result: ClusterCapacity | undefined; + service.getClusterCapacity().subscribe(r => (result = r)); + + const req = httpMock.expectOne(`${api}/computing-unit/admin/cluster-capacity`); + expect(req.request.method).toEqual("GET"); + req.flush(capacity); + + expect(result).toEqual(capacity); + }); + + it("getClusterCapacity() surfaces the 404 a deployment without Kubernetes answers", () => { + let status: number | undefined; + service.getClusterCapacity().subscribe({ error: (e: unknown) => (status = (e as HttpErrorResponse).status) }); + + httpMock + .expectOne(`${api}/computing-unit/admin/cluster-capacity`) + .flush({ message: "not on Kubernetes" }, { status: 404, statusText: "Not Found" }); + + expect(status).toBe(404); + }); + it("renameComputingUnit() PUTs to a URI-encoded rename endpoint", () => { service.renameComputingUnit(3, "my unit/name").subscribe(); diff --git a/frontend/src/app/common/service/computing-unit/workflow-computing-unit/workflow-computing-unit-managing.service.ts b/frontend/src/app/common/service/computing-unit/workflow-computing-unit/workflow-computing-unit-managing.service.ts index 6c4c1f01385..c752b59b458 100644 --- a/frontend/src/app/common/service/computing-unit/workflow-computing-unit/workflow-computing-unit-managing.service.ts +++ b/frontend/src/app/common/service/computing-unit/workflow-computing-unit/workflow-computing-unit-managing.service.ts @@ -22,6 +22,7 @@ import { HttpClient } from "@angular/common/http"; import { Observable } from "rxjs"; import { AppSettings } from "../../../app-setting"; import { + ClusterCapacity, DashboardWorkflowComputingUnit, WorkflowComputingUnit, WorkflowComputingUnitResourceLimit, @@ -34,6 +35,7 @@ export const COMPUTING_UNIT_CREATE_URL = `${COMPUTING_UNIT_BASE_URL}/create`; export const COMPUTING_UNIT_LIST_URL = `${COMPUTING_UNIT_BASE_URL}`; export const COMPUTING_UNIT_TYPES_URL = `${COMPUTING_UNIT_BASE_URL}/types`; export const COMPUTING_UNIT_ADMIN_LIST_URL = `${COMPUTING_UNIT_BASE_URL}/admin/list`; +export const COMPUTING_UNIT_ADMIN_CLUSTER_CAPACITY_URL = `${COMPUTING_UNIT_BASE_URL}/admin/cluster-capacity`; @Injectable({ providedIn: "root", @@ -200,6 +202,13 @@ export class WorkflowComputingUnitManagingService { .pipe(map(arr => arr.map(unit => this.parseDashboardUnit(unit)))); } + /** The cluster's CPU, memory and GPU capacity. Admin only, and 404 when units do not run on Kubernetes. */ + public getClusterCapacity(): Observable { + return this.http.get( + `${AppSettings.getApiEndpoint()}/${COMPUTING_UNIT_ADMIN_CLUSTER_CAPACITY_URL}` + ); + } + public getComputingUnit(cuid: number): Observable { return this.http .get(`${AppSettings.getApiEndpoint()}/${COMPUTING_UNIT_BASE_URL}/${cuid}`) diff --git a/frontend/src/app/common/type/workflow-computing-unit.ts b/frontend/src/app/common/type/workflow-computing-unit.ts index a883b39200c..53650d68b35 100644 --- a/frontend/src/app/common/type/workflow-computing-unit.ts +++ b/frontend/src/app/common/type/workflow-computing-unit.ts @@ -55,3 +55,16 @@ export interface DashboardWorkflowComputingUnit { ownerAvatar: string; ownerName: string; } + +// CPU in millicores, memory in bytes, GPU in devices. +export interface ResourceCapacity { + total: number; + allocated: number; + available: number; +} + +export interface ClusterCapacity { + cpu: ResourceCapacity; + memory: ResourceCapacity; + gpu: ResourceCapacity; +} diff --git a/frontend/src/app/dashboard/component/admin/computing-unit/admin-computing-unit.component.html b/frontend/src/app/dashboard/component/admin/computing-unit/admin-computing-unit.component.html index d0faa5b6f4d..3182ade0141 100644 --- a/frontend/src/app/dashboard/component/admin/computing-unit/admin-computing-unit.component.html +++ b/frontend/src/app/dashboard/component/admin/computing-unit/admin-computing-unit.component.html @@ -27,6 +27,8 @@ + + { service = TestBed.inject(WorkflowComputingUnitManagingService); // No `detectChanges()` here, so the poll starts only in the tests that ask for it. vi.spyOn(service, "listAllComputingUnits").mockReturnValue(of([])); + // Never answers, so the capacity panel stays hidden and adds no alert here. + vi.spyOn(service, "getClusterCapacity").mockReturnValue(NEVER); }); afterEach(() => { @@ -143,6 +145,15 @@ describe("AdminComputingUnitComponent", () => { expect(rows[0].textContent).toContain("alice"); }); + it("places the cluster capacity panel above the units table", () => { + fixture.detectChanges(); + + const panel = fixture.nativeElement.querySelector("texera-admin-cluster-capacity"); + const table = fixture.nativeElement.querySelector("nz-table"); + expect(panel).not.toBeNull(); + expect(panel.compareDocumentPosition(table) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy(); + }); + describe("loading and polling", () => { // Fake timers go in before `ngOnInit`, so the poll's interval is fake too. beforeEach(() => { diff --git a/frontend/src/app/dashboard/component/admin/computing-unit/admin-computing-unit.component.ts b/frontend/src/app/dashboard/component/admin/computing-unit/admin-computing-unit.component.ts index 3ff5f5c901f..6d5d2571413 100644 --- a/frontend/src/app/dashboard/component/admin/computing-unit/admin-computing-unit.component.ts +++ b/frontend/src/app/dashboard/component/admin/computing-unit/admin-computing-unit.component.ts @@ -57,6 +57,7 @@ import { formatRelativeTime } from "../../../../common/util/format.util"; import { extractErrorMessage } from "../../../../common/util/error"; import { UserAvatarComponent } from "../../user/user-avatar/user-avatar.component"; import { ComputingUnitState } from "../../../../common/type/computing-unit-connection.interface"; +import { AdminClusterCapacityComponent } from "./cluster-capacity/admin-cluster-capacity.component"; // Same cadence as the admin executions page. const COMPUTING_UNIT_REFRESH_INTERVAL_MS = 5000; @@ -90,6 +91,7 @@ type SpecKey = Exclude + + + + + + + + + + + + + + + + + + + + + + + + + +
Resource + Share allocated + AllocatedAvailableTotal
{{ row.resource }} +
+ + {{ row.percentUsed }}% + + + + + + +
+ + {{ row.note }} + + 		+ {{ row.allocated }} + + {{ row.available }} + + {{ row.total }} +
+

+ Allocated counts what running pods request, not what they use. Available is summed across nodes, so a unit too big + for any one node's free space still waits in Pending. +

+ + + + + + + Updated {{ lastUpdated | date: "mediumTime" }} + + + + diff --git a/frontend/src/app/dashboard/component/admin/computing-unit/cluster-capacity/admin-cluster-capacity.component.scss b/frontend/src/app/dashboard/component/admin/computing-unit/cluster-capacity/admin-cluster-capacity.component.scss new file mode 100644 index 00000000000..246f3dee9e2 --- /dev/null +++ b/frontend/src/app/dashboard/component/admin/computing-unit/cluster-capacity/admin-cluster-capacity.component.scss @@ -0,0 +1,194 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +$ink-muted: rgba(0, 0, 0, 0.45); +$hairline: #f0f0f0; + +// A container, so the ledger stacks by the card's width, which the sidebar decides, not the window's. +.cluster-capacity { + container: cluster-capacity / inline-size; + // The track is a lighter step of the fill's hue, so a flagged row reads as one band. + --meter-fill: #1890ff; + --meter-track: #bae7ff; + + margin-bottom: 16px; +} + +.capacity-extra { + display: flex; + align-items: center; + gap: 8px; +} + +.capacity-updated { + font-size: 12px; + color: $ink-muted; +} + +.capacity-alert { + margin-bottom: 8px; +} + +.capacity-ledger { + width: 100%; + + th, + td { + padding: 8px 0 8px 24px; + text-align: right; + white-space: nowrap; + } + + th:first-child { + padding-left: 0; + text-align: left; + } + + thead th { + padding-top: 0; + font-weight: 400; + font-size: 12px; + color: $ink-muted; + border-bottom: 1px solid $hairline; + } + + tbody th { + font-weight: 500; + } + + // Takes the slack, so the bars line up across rows. + .usage { + width: 100%; + text-align: left; + } +} + +.capacity-warning { + --meter-fill: #faad14; + --meter-track: #fff1b8; +} + +.capacity-critical { + --meter-fill: #ff4d4f; + --meter-track: #ffccc7; +} + +.meter { + display: flex; + align-items: center; + gap: 12px; + min-width: 160px; +} + +// Square where it starts, rounded where the data ends. +.meter-track, +.meter-fill { + border-radius: 0 4px 4px 0; +} + +.meter-track { + flex: 1; + height: 8px; + background: var(--meter-track); +} + +.meter-fill { + height: 100%; + background: var(--meter-fill); +} + +// Fixed widths, so a status icon does not shorten its row's bar. +.percent-used { + min-width: 4ch; + text-align: right; + font-weight: 500; +} + +.status-slot { + display: inline-flex; + width: 16px; +} + +.status-icon { + color: var(--meter-fill); +} + +.meter-note { + color: $ink-muted; +} + +.capacity-note { + max-width: 72ch; + margin: 8px 0 0; + font-size: 12px; + color: $ink-muted; +} + +@media (prefers-reduced-motion: reduce) { + .refresh-button .anticon-spin { + animation: none; + } +} + +// Too narrow for the ledger, so each resource stacks into labelled lines. +@container cluster-capacity (max-width: 545px) { + .capacity-ledger { + thead { + display: none; + } + + tbody tr { + display: flex; + flex-direction: column; + gap: 4px; + padding: 8px 0; + } + + tbody tr:not(:last-child) { + border-bottom: 1px solid $hairline; + } + + th, + td { + padding: 0; + } + + td[data-label] { + display: flex; + justify-content: space-between; + gap: 12px; + } + + td[data-label]::before { + content: attr(data-label); + color: $ink-muted; + } + } + + .meter { + min-width: 0; + } +} + +// The card squeezes its title before its extra, so the timestamp goes instead. +@container cluster-capacity (max-width: 320px) { + .capacity-updated { + display: none; + } +} diff --git a/frontend/src/app/dashboard/component/admin/computing-unit/cluster-capacity/admin-cluster-capacity.component.spec.ts b/frontend/src/app/dashboard/component/admin/computing-unit/cluster-capacity/admin-cluster-capacity.component.spec.ts new file mode 100644 index 00000000000..da65998601a --- /dev/null +++ b/frontend/src/app/dashboard/component/admin/computing-unit/cluster-capacity/admin-cluster-capacity.component.spec.ts @@ -0,0 +1,346 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { HttpErrorResponse } from "@angular/common/http"; +import { ComponentFixture, TestBed } from "@angular/core/testing"; +import { of, Subject, throwError } from "rxjs"; +import { AdminClusterCapacityComponent } from "./admin-cluster-capacity.component"; +import { WorkflowComputingUnitManagingService } from "../../../../../common/service/computing-unit/workflow-computing-unit/workflow-computing-unit-managing.service"; +import { ClusterCapacity, ResourceCapacity } from "../../../../../common/type/workflow-computing-unit"; +import { commonTestImports, commonTestProviders } from "../../../../../common/testing/test-utils"; + +const MiB = 1024 ** 2; +const GiB = 1024 ** 3; +const TiB = 1024 ** 4; + +// A quarter of every resource allocated, so each row reads 25%. +function capacity(over: Partial = {}): ClusterCapacity { + return { + cpu: used(16000, 4000), + memory: used(64 * GiB, 16 * GiB), + gpu: used(4, 1), + ...over, + }; +} + +const used = (total: number, allocated: number): ResourceCapacity => ({ + total, + allocated, + available: Math.max(total - allocated, 0), +}); + +const failWith = (status: number, message: string) => + throwError(() => new HttpErrorResponse({ error: { message }, status })); + +describe("AdminClusterCapacityComponent", () => { + let fixture: ComponentFixture; + let service: WorkflowComputingUnitManagingService; + + beforeEach(async () => { + await TestBed.configureTestingModule({ + providers: [...commonTestProviders], + imports: [AdminClusterCapacityComponent, ...commonTestImports], + }).compileComponents(); + + // Fake timers go in before the first `detectChanges`, so the poll's interval is fake too. + vi.useFakeTimers(); + fixture = TestBed.createComponent(AdminClusterCapacityComponent); + service = TestBed.inject(WorkflowComputingUnitManagingService); + vi.spyOn(service, "getClusterCapacity").mockReturnValue(of(capacity())); + }); + + afterEach(() => { + fixture.destroy(); + vi.useRealTimers(); + vi.restoreAllMocks(); + }); + + const render = () => fixture.detectChanges(); + const advance = (ms: number) => { + vi.advanceTimersByTime(ms); + fixture.detectChanges(); + }; + const panel = (): HTMLElement | null => fixture.nativeElement.querySelector(".cluster-capacity"); + const rowFor = (resource: string): HTMLElement => { + const row = Array.from(fixture.nativeElement.querySelectorAll("tbody tr")).find( + r => r.querySelector("th")?.textContent?.trim() === resource + ); + expect(row).toBeDefined(); + return row!; + }; + const text = (resource: string, cell: string) => rowFor(resource).querySelector(`.${cell}`)?.textContent?.trim(); + const figures = (resource: string) => ["allocated", "available", "total"].map(cell => text(resource, cell)); + const fillWidth = (resource: string) => + rowFor(resource).querySelector(".meter-fill")?.style.width ?? "no meter"; + const alertText = () => fixture.nativeElement.querySelector("nz-alert")?.textContent ?? ""; + + describe("the ledger", () => { + it("shows CPU in cores, memory in binary units and GPU as a count, each with its share allocated", () => { + render(); + + expect(figures("CPU")).toEqual(["4 cores", "12 cores", "16 cores"]); + expect(figures("Memory")).toEqual(["16 GiB", "48 GiB", "64 GiB"]); + expect(figures("GPU")).toEqual(["1", "3", "4"]); + for (const resource of ["CPU", "Memory", "GPU"]) { + expect(text(resource, "percent-used")).toBe("25%"); + expect(fillWidth(resource)).toBe("25%"); + } + expect(panel()?.textContent).toContain("Updated"); + }); + + it("says 1 core in the singular and groups the thousands", () => { + vi.mocked(service.getClusterCapacity).mockReturnValue( + of(capacity({ cpu: { total: 2_048_000, allocated: 250, available: 1000 } })) + ); + + render(); + + expect(text("CPU", "available")).toBe("1 core"); + expect(text("CPU", "total")).toBe("2,048 cores"); + }); + + // 999m is just under a core, the figure that would round to a plural "1". + it("shows cores to the millicore, without rounding", () => { + vi.mocked(service.getClusterCapacity).mockReturnValue( + of(capacity({ cpu: { total: 3920, allocated: 125, available: 999 } })) + ); + + render(); + + expect(text("CPU", "allocated")).toBe("0.125 cores"); + expect(text("CPU", "available")).toBe("0.999 cores"); + expect(text("CPU", "total")).toBe("3.92 cores"); + }); + + it("picks the memory unit by size and drops a trailing .0", () => { + vi.mocked(service.getClusterCapacity).mockReturnValue( + of(capacity({ memory: { total: 2 * TiB, allocated: 512 * MiB, available: 1.5 * GiB } })) + ); + + render(); + + expect(text("Memory", "allocated")).toBe("512 MiB"); + expect(text("Memory", "available")).toBe("1.5 GiB"); + expect(text("Memory", "total")).toBe("2 TiB"); + }); + + it("moves up a unit when rounding reaches it, so it never shows 1,024 of the smaller one", () => { + vi.mocked(service.getClusterCapacity).mockReturnValue(of(capacity({ memory: used(GiB - 1, 1023.99 * MiB) }))); + + render(); + + expect(text("Memory", "allocated")).toBe("1 GiB"); + expect(text("Memory", "total")).toBe("1 GiB"); + }); + + it("shows zero bytes as 0 B", () => { + vi.mocked(service.getClusterCapacity).mockReturnValue(of(capacity({ memory: used(64 * GiB, 0) }))); + + render(); + + expect(text("Memory", "allocated")).toBe("0 B"); + expect(text("Memory", "percent-used")).toBe("0%"); + }); + + // 15.99 of 16 cores must not read full, and 29 / 100 * 100 is 28.999… in floating point. + it("rounds the share down, so 100% shows only when nothing is left", () => { + vi.mocked(service.getClusterCapacity).mockReturnValue( + of(capacity({ cpu: used(16000, 15990), memory: used(64 * GiB, 64 * GiB), gpu: used(100, 29) })) + ); + + render(); + + expect(text("CPU", "percent-used")).toBe("99%"); + expect(text("Memory", "percent-used")).toBe("100%"); + expect(text("GPU", "percent-used")).toBe("29%"); + }); + + it("shows a share over 100% when pods request more than the nodes offer, with the bar capped at full", () => { + vi.mocked(service.getClusterCapacity).mockReturnValue(of(capacity({ cpu: used(1000, 1500) }))); + + render(); + + expect(text("CPU", "percent-used")).toBe("150%"); + expect(fillWidth("CPU")).toBe("100%"); + expect(text("CPU", "available")).toBe("0 cores"); + }); + + it("flags a resource from 80% as running low and from 95% as nearly full, with an icon and a label", () => { + vi.mocked(service.getClusterCapacity).mockReturnValue( + of(capacity({ cpu: used(100, 79), memory: used(100, 80), gpu: used(20, 19) })) + ); + + render(); + + const status = (resource: string) => rowFor(resource).querySelector(".status-icon")?.getAttribute("aria-label"); + expect(rowFor("CPU").className).not.toMatch(/capacity-(warning|critical)/); + expect(status("CPU")).toBeUndefined(); + expect(rowFor("Memory").classList).toContain("capacity-warning"); + expect(status("Memory")).toBe("Running low"); + expect(rowFor("GPU").classList).toContain("capacity-critical"); + expect(status("GPU")).toBe("Nearly full"); + }); + + it("says the cluster has none of a resource instead of dividing by zero", () => { + vi.mocked(service.getClusterCapacity).mockReturnValue(of(capacity({ gpu: used(0, 0) }))); + + render(); + + expect(text("GPU", "usage")).toBe("None in the cluster"); + expect(text("GPU", "percent-used")).toBeUndefined(); + expect(fillWidth("GPU")).toBe("no meter"); + expect(text("GPU", "total")).toBe("0"); + }); + + it("shows a resource the response leaves out or garbles as not reported, and still shows the rest", () => { + const garbled = { + cpu: { total: "lots", allocated: 1, available: 1 }, + memory: capacity().memory, + } as unknown as ClusterCapacity; + vi.mocked(service.getClusterCapacity).mockReturnValue(of(garbled)); + + render(); + + for (const resource of ["CPU", "GPU"]) { + expect(text(resource, "usage")).toBe("Not reported"); + expect(figures(resource)).toEqual(["—", "—", "—"]); + } + expect(text("Memory", "percent-used")).toBe("25%"); + }); + + // A browser drops a negative width, so the bar would show full. + it("shows a resource with a negative figure as not reported", () => { + vi.mocked(service.getClusterCapacity).mockReturnValue( + of( + capacity({ + cpu: { total: 1000, allocated: -50, available: 1050 }, + gpu: { total: -1, allocated: 0, available: 0 }, + }) + ) + ); + + render(); + + expect(text("CPU", "usage")).toBe("Not reported"); + expect(text("GPU", "usage")).toBe("Not reported"); + }); + }); + + describe("loading and polling", () => { + it("renders nothing until the first answer", () => { + const first = new Subject(); + vi.mocked(service.getClusterCapacity).mockReturnValue(first); + + render(); + expect(panel()).toBeNull(); + + first.next(capacity()); + first.complete(); + render(); + expect(panel()).not.toBeNull(); + }); + + it("stays hidden and stops polling when computing units do not run on Kubernetes", () => { + vi.mocked(service.getClusterCapacity).mockReturnValue(failWith(404, "not on Kubernetes")); + + render(); + advance(120_000); + + expect(panel()).toBeNull(); + expect(service.getClusterCapacity).toHaveBeenCalledTimes(1); + }); + + it("shows the cluster's message when the first load fails, e.g. a missing read grant", () => { + vi.mocked(service.getClusterCapacity).mockReturnValue(failWith(500, "nodes is forbidden")); + + render(); + + expect(panel()).not.toBeNull(); + expect(alertText()).toContain("Could not load cluster capacity: nodes is forbidden"); + expect(fixture.nativeElement.querySelector("table")).toBeNull(); + }); + + // An empty message is falsy, and would leave the alert ending on a bare colon. + it("still reports a failure whose message is empty, and says no details were given", () => { + vi.mocked(service.getClusterCapacity) + .mockReturnValueOnce(failWith(500, "")) + .mockReturnValueOnce(of(capacity())) + .mockReturnValueOnce(failWith(500, "")); + + render(); + expect(alertText()).toContain("Could not load cluster capacity: the server gave no details"); + + advance(30_000); + advance(30_000); + expect(alertText()).toContain("The refresh is failing: the server gave no details"); + }); + + it("refreshes every 30 seconds, not on the units table's 5 second cadence", () => { + render(); + advance(29_999); + expect(service.getClusterCapacity).toHaveBeenCalledTimes(1); + + advance(1); + expect(service.getClusterCapacity).toHaveBeenCalledTimes(2); + }); + + it("keeps the last capacity and warns when a refresh fails, then clears the warning once one succeeds", () => { + vi.mocked(service.getClusterCapacity) + .mockReturnValueOnce(of(capacity())) + .mockReturnValueOnce(failWith(500, "boom")) + .mockReturnValueOnce(of(capacity({ cpu: used(16000, 8000) }))); + + render(); + advance(30_000); + + expect(text("CPU", "percent-used")).toBe("25%"); + expect(alertText()).toContain("The refresh is failing: boom"); + + advance(30_000); + + expect(text("CPU", "percent-used")).toBe("50%"); + expect(fixture.nativeElement.querySelector("nz-alert")).toBeNull(); + }); + + it("restarts the 30 second clock when the admin clicks refresh", () => { + render(); + advance(20_000); + fixture.nativeElement.querySelector(".refresh-button").click(); + expect(service.getClusterCapacity).toHaveBeenCalledTimes(2); + + advance(29_999); + expect(service.getClusterCapacity).toHaveBeenCalledTimes(2); + + advance(1); + expect(service.getClusterCapacity).toHaveBeenCalledTimes(3); + }); + + it("ignores a refresh click while a fetch is still in flight", () => { + const inFlight = new Subject(); + vi.mocked(service.getClusterCapacity).mockReturnValueOnce(of(capacity())).mockReturnValueOnce(inFlight); + + render(); + const button = fixture.nativeElement.querySelector(".refresh-button"); + button.click(); + button.click(); + + expect(service.getClusterCapacity).toHaveBeenCalledTimes(2); + }); + }); +}); diff --git a/frontend/src/app/dashboard/component/admin/computing-unit/cluster-capacity/admin-cluster-capacity.component.ts b/frontend/src/app/dashboard/component/admin/computing-unit/cluster-capacity/admin-cluster-capacity.component.ts new file mode 100644 index 00000000000..b1fa5f0216c --- /dev/null +++ b/frontend/src/app/dashboard/component/admin/computing-unit/cluster-capacity/admin-cluster-capacity.component.ts @@ -0,0 +1,191 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Component, OnInit } from "@angular/core"; +import { HttpErrorResponse } from "@angular/common/http"; +import { DatePipe, NgFor, NgIf } from "@angular/common"; +import { UntilDestroy, untilDestroyed } from "@ngneat/until-destroy"; +import { EMPTY, interval, Subject, throwError } from "rxjs"; +import { catchError, exhaustMap, finalize, startWith, switchMap } from "rxjs/operators"; +import { NzAlertComponent } from "ng-zorro-antd/alert"; +import { NzButtonComponent } from "ng-zorro-antd/button"; +import { NzCardComponent } from "ng-zorro-antd/card"; +import { NzIconDirective } from "ng-zorro-antd/icon"; +import { NzTooltipDirective } from "ng-zorro-antd/tooltip"; +import { WorkflowComputingUnitManagingService } from "../../../../../common/service/computing-unit/workflow-computing-unit/workflow-computing-unit-managing.service"; +import { ResourceCapacity } from "../../../../../common/type/workflow-computing-unit"; +import { extractErrorMessage } from "../../../../../common/util/error"; + +// Capacity moves only when units start or stop, and each fetch lists every pod, so this polls slower than the table. +const CLUSTER_CAPACITY_REFRESH_INTERVAL_MS = 30_000; + +const MILLICORES_PER_CORE = 1000; +const BYTES_PER_UNIT = 1024; +const BYTE_UNITS = ["B", "KiB", "MiB", "GiB", "TiB", "PiB"]; + +// Three decimals show millicores exactly. +const exactFormat = new Intl.NumberFormat("en-US", { maximumFractionDigits: 3 }); +const bytesFormat = new Intl.NumberFormat("en-US", { maximumFractionDigits: 1 }); + +interface CapacityStatus { + minPercent: number; + label: string; + icon: string; + className: string; +} + +// Compared with the share as shown, so 94% is never flagged as nearly full. +const CAPACITY_STATUSES: CapacityStatus[] = [ + { minPercent: 95, label: "Nearly full", icon: "warning", className: "capacity-critical" }, + { minPercent: 80, label: "Running low", icon: "exclamation-circle", className: "capacity-warning" }, +]; + +/** `percentUsed` is unset when there is no total to divide by, and `note` then says why. */ +interface CapacityRow { + resource: string; + allocated: string; + available: string; + total: string; + percentUsed?: number; + fillPercent?: number; + status?: CapacityStatus; + note?: string; +} + +const formatCores = (millicores: number): string => { + const cores = exactFormat.format(millicores / MILLICORES_PER_CORE); + return `${cores} ${cores === "1" ? "core" : "cores"}`; +}; + +/** Binary units, as Kubernetes quotes memory. */ +const formatBytes = (bytes: number): string => { + let value = bytes; + let unit = 0; + // Compared as shown, so 1023.99 MiB reads 1 GiB, not 1,024 MiB. + while (Math.round(value * 10) / 10 >= BYTES_PER_UNIT && unit < BYTE_UNITS.length - 1) { + value /= BYTES_PER_UNIT; + unit++; + } + return `${bytesFormat.format(value)} ${BYTE_UNITS[unit]}`; +}; + +const formatGpus = (count: number): string => exactFormat.format(count); + +const isAmount = (n: number): boolean => Number.isFinite(n) && n >= 0; + +function toRow(resource: string, capacity: ResourceCapacity | undefined, format: (n: number) => string): CapacityRow { + // A server response, so a missing, garbled or negative figure gets a row of its own. + if (!capacity || ![capacity.total, capacity.allocated, capacity.available].every(isAmount)) { + return { resource, allocated: "—", available: "—", total: "—", note: "Not reported" }; + } + const { total, allocated, available } = capacity; + const values = { resource, allocated: format(allocated), available: format(available), total: format(total) }; + if (total === 0) { + return { ...values, note: "None in the cluster" }; + } + // Rounded down, so 100% means nothing is left. Multiplying first keeps 29 of 100 from flooring to 28%. + const percentUsed = Math.floor((allocated * 100) / total); + return { + ...values, + percentUsed, + // `allocated` can exceed `total`, so the share can pass 100% but the bar cannot. + fillPercent: Math.min(allocated / total, 1) * 100, + status: CAPACITY_STATUSES.find(s => percentUsed >= s.minPercent), + }; +} + +@UntilDestroy() +@Component({ + selector: "texera-admin-cluster-capacity", + templateUrl: "./admin-cluster-capacity.component.html", + styleUrls: ["./admin-cluster-capacity.component.scss"], + imports: [ + NzAlertComponent, + NzButtonComponent, + NzCardComponent, + NzIconDirective, + NzTooltipDirective, + NgFor, + NgIf, + DatePipe, + ], +}) +export class AdminClusterCapacityComponent implements OnInit { + rows?: CapacityRow[]; + // Set by a 404: units do not run on Kubernetes, so there is no cluster to show. + notOnKubernetes = false; + loadError?: string; + lastUpdated?: Date; + isFetching = false; + private readonly refreshClicks = new Subject(); + + constructor(private computingUnitService: WorkflowComputingUnitManagingService) {} + + // Hidden until the first answer, so a deployment without Kubernetes never flashes the panel. + get isVisible(): boolean { + return !this.notOnKubernetes && (this.rows !== undefined || this.loadError !== undefined); + } + + ngOnInit(): void { + // A click restarts the clock. A 404 cannot change while the service runs, so it alone passes the inner + // `catchError`, and the outer one ends the poll. + this.refreshClicks + .pipe( + startWith(undefined), + // Not `timer(0, …)`, which would delay the fetch to the next macrotask. + switchMap(() => interval(CLUSTER_CAPACITY_REFRESH_INTERVAL_MS).pipe(startWith(0))), + exhaustMap(() => { + this.isFetching = true; + return this.computingUnitService.getClusterCapacity().pipe( + catchError((err: unknown) => { + if (err instanceof HttpErrorResponse && err.status === 404) { + return throwError(() => err); + } + // The alerts end on this message, so it cannot be empty. + this.loadError = extractErrorMessage(err) || "the server gave no details"; + return EMPTY; + }), + finalize(() => (this.isFetching = false)) + ); + }), + catchError(() => { + this.notOnKubernetes = true; + return EMPTY; + }), + untilDestroyed(this) + ) + .subscribe(capacity => { + this.loadError = undefined; + this.lastUpdated = new Date(); + this.rows = [ + toRow("CPU", capacity?.cpu, formatCores), + toRow("Memory", capacity?.memory, formatBytes), + toRow("GPU", capacity?.gpu, formatGpus), + ]; + }); + } + + refresh(): void { + this.refreshClicks.next(); + } + + trackByResource(_index: number, row: CapacityRow): string { + return row.resource; + } +}