diff --git a/app/api/assessment/datasets/[dataset_id]/route.ts b/app/api/assessment/datasets/[dataset_id]/route.ts index 7296408..a4f035b 100644 --- a/app/api/assessment/datasets/[dataset_id]/route.ts +++ b/app/api/assessment/datasets/[dataset_id]/route.ts @@ -1,58 +1,19 @@ -// BFF proxy — GET (with optional S3 file fetch, max 10 MB) + DELETE /api/v1/assessment/datasets/:id +// BFF proxy — GET (optional preview via limit_rows) + DELETE /api/v1/assessment/datasets/:id import { NextRequest, NextResponse } from "next/server"; import { apiClient } from "@/app/lib/apiClient"; import { proxyErrorResponse, withQueryParams } from "@/app/api/_routeProxy"; -const MAX_DATASET_PROXY_BYTES = 10 * 1024 * 1024; - -async function readFileAsBase64WithLimit(response: Response): Promise { - const contentLength = response.headers.get("content-length"); - if (contentLength) { - const size = Number.parseInt(contentLength, 10); - if (Number.isFinite(size) && size > MAX_DATASET_PROXY_BYTES) { - throw new Error("FILE_TOO_LARGE"); - } - } - - const reader = response.body?.getReader(); - if (!reader) { - throw new Error("FILE_STREAM_UNAVAILABLE"); - } - - const chunks: Uint8Array[] = []; - let totalBytes = 0; - - while (true) { - const { done, value } = await reader.read(); - if (done) break; - if (!value) continue; - - totalBytes += value.byteLength; - if (totalBytes > MAX_DATASET_PROXY_BYTES) { - throw new Error("FILE_TOO_LARGE"); - } - chunks.push(value); - } - - return Buffer.concat(chunks).toString("base64"); -} - export async function GET( request: NextRequest, { params }: { params: Promise<{ dataset_id: string }> }, ) { try { const { dataset_id } = await params; - const fetchContent = - request.nextUrl.searchParams.get("fetch_content") === "true"; + const limitRows = request.nextUrl.searchParams.get("limit_rows"); - // Always request signed URL when fetch_content is needed const backendParams = new URLSearchParams(); - if (fetchContent) { - backendParams.set("fetch_content", "true"); - } - if (fetchContent) { - backendParams.set("include_signed_url", "true"); + if (limitRows) { + backendParams.set("limit_rows", limitRows); } const endpoint = withQueryParams( `/api/v1/assessment/datasets/${dataset_id}`, @@ -63,56 +24,6 @@ export async function GET( method: "GET", }); - if (status >= 400) { - return NextResponse.json(data, { status }); - } - - // Download file from S3 server-side and return as base64 - if (fetchContent) { - const signedUrl = - (data as { data?: { signed_url?: string }; signed_url?: string })?.data - ?.signed_url || - (data as { data?: { signed_url?: string }; signed_url?: string }) - ?.signed_url; - - if (!signedUrl) { - return NextResponse.json( - { error: "No signed URL available" }, - { status: 404 }, - ); - } - - const fileResponse = await fetch(signedUrl); - if (!fileResponse.ok) { - return NextResponse.json( - { error: "Failed to fetch file from storage" }, - { status: 502 }, - ); - } - - let base64: string; - try { - base64 = await readFileAsBase64WithLimit(fileResponse); - } catch (error) { - if (error instanceof Error && error.message === "FILE_TOO_LARGE") { - return NextResponse.json( - { error: "File too large" }, - { status: 413 }, - ); - } - - return NextResponse.json( - { error: "Failed to read file from storage" }, - { status: 502 }, - ); - } - - return NextResponse.json( - { ...(data as Record), file_content: base64 }, - { status: 200 }, - ); - } - return NextResponse.json(data, { status }); } catch (error: unknown) { return proxyErrorResponse("Assessment dataset details proxy error:", error); diff --git a/app/components/assessment/ColumnMapperStep.tsx b/app/components/assessment/ColumnMapperStep.tsx index 41a09a1..0fa075d 100644 --- a/app/components/assessment/ColumnMapperStep.tsx +++ b/app/components/assessment/ColumnMapperStep.tsx @@ -158,19 +158,19 @@ export default function ColumnMapperStep({
-
-
-
+
+
+
- + {column}
-
+
{ASSESSMENT_ROLE_OPTIONS.map((option) => { const isGroundTruth = option.value === "ground_truth"; const isActive = config.role === option.value; diff --git a/app/components/assessment/DataViewModal.tsx b/app/components/assessment/DataViewModal.tsx index 6e2fd19..74b04de 100644 --- a/app/components/assessment/DataViewModal.tsx +++ b/app/components/assessment/DataViewModal.tsx @@ -1,5 +1,6 @@ "use client"; +import { useMemo } from "react"; import { Modal } from "@/app/components"; import CloseIcon from "@/app/components/icons/document/CloseIcon"; interface DataViewModalProps { @@ -10,16 +11,29 @@ interface DataViewModalProps { onClose: () => void; } -/** - * Reusable modal for viewing tabular data (dataset preview, result preview). - */ +const isBlank = (cell: string | undefined) => + cell == null || String(cell).trim() === ""; + export default function DataViewModal({ title, subtitle, - headers, - rows, + headers: rawHeaders, + rows: rawRows, onClose, }: DataViewModalProps) { + const { headers, rows } = useMemo(() => { + const keptColIdx = rawHeaders + .map((_, colIdx) => colIdx) + .filter((colIdx) => rawRows.some((row) => !isBlank(row[colIdx]))); + + const filteredHeaders = keptColIdx.map((idx) => rawHeaders[idx]); + const filteredRows = rawRows + .map((row) => keptColIdx.map((idx) => row[idx] ?? "")) + .filter((row) => row.some((cell) => !isBlank(cell))); + + return { headers: filteredHeaders, rows: filteredRows }; + }, [rawHeaders, rawRows]); + return (
- +
- + {headers.map((header, i) => ( @@ -66,8 +80,11 @@ export default function DataViewModal({ {rowIdx + 1} {row.map((cell, cellIdx) => ( - diff --git a/app/components/assessment/DatasetsTab.tsx b/app/components/assessment/DatasetsTab.tsx index e54aa2b..b3fcf45 100644 --- a/app/components/assessment/DatasetsTab.tsx +++ b/app/components/assessment/DatasetsTab.tsx @@ -7,12 +7,18 @@ import { useToast } from "@/app/components/Toast"; import { useAuth } from "@/app/lib/context/AuthContext"; import { extractCreatedDataset, - fetchAndParseDatasetFile, + fetchDatasetPreview, handleForbiddenError, isAllowedDatasetFile, } from "@/app/lib/utils/assessment"; +import { PREVIEW_ROW_LIMIT } from "@/app/lib/assessment/results"; +import { + DATASET_SAMPLE_ROW_LIMIT, + MAX_DATASET_FILE_BYTES, +} from "@/app/lib/assessment/constants"; import type { CreateDatasetResponse, + DatasetPreview, DatasetResponse, DatasetsTabProps, DatasetViewModalData, @@ -49,6 +55,9 @@ export default function DatasetsTab({ useState(null); const [confirmDeleteId, setConfirmDeleteId] = useState(null); const [deletingId, setDeletingId] = useState(null); + const [previewCache, setPreviewCache] = useState< + Record + >({}); const loadDatasets = useCallback(async () => { if (!isAuthenticated) return; @@ -91,6 +100,14 @@ export default function DatasetsTab({ return; } + if (file.size > MAX_DATASET_FILE_BYTES) { + toast.error( + `File too large. Max ${MAX_DATASET_FILE_BYTES / (1024 * 1024)}MB allowed.`, + ); + event.target.value = ""; + return; + } + setUploadedFile(file); if (!datasetName) { setDatasetName(file.name.replace(/\.(csv|xlsx|xls)$/i, "")); @@ -163,15 +180,28 @@ export default function DatasetsTab({ setIsLoadingColumns(true); try { - const parsed = await fetchAndParseDatasetFile(id, apiKey); + const cached = previewCache[id]; + const parsed = + cached ?? + (await fetchDatasetPreview(id, apiKey, DATASET_SAMPLE_ROW_LIMIT)); + if (!cached) { + setPreviewCache((prev) => ({ ...prev, [id]: parsed })); + } + const isBlank = (cell: string | undefined) => + cell == null || String(cell).trim() === ""; + const keptIdx = parsed.headers + .map((_, colIdx) => colIdx) + .filter((colIdx) => parsed.rows.some((row) => !isBlank(row[colIdx]))); + + const filteredHeaders = keptIdx.map((idx) => parsed.headers[idx]); const firstRow = parsed.rows[0] || []; const sampleRow = Object.fromEntries( - parsed.headers.map((header, index) => [ - header, - String(firstRow[index] ?? ""), + keptIdx.map((idx) => [ + parsed.headers[idx], + String(firstRow[idx] ?? ""), ]), ); - onColumnsLoaded(parsed.headers, sampleRow); + onColumnsLoaded(filteredHeaders, sampleRow); } catch (error) { if (handleForbiddenError(error, onForbidden)) return; const message = @@ -188,9 +218,25 @@ export default function DatasetsTab({ }; const handleViewDataset = async (selectedDatasetId: number, name: string) => { + const cacheKey = String(selectedDatasetId); + const cached = previewCache[cacheKey]; + if (cached) { + setViewModalData({ + name, + headers: cached.headers, + rows: cached.rows, + }); + return; + } + setViewingId(selectedDatasetId); try { - const parsed = await fetchAndParseDatasetFile(selectedDatasetId, apiKey); + const parsed = await fetchDatasetPreview( + selectedDatasetId, + apiKey, + PREVIEW_ROW_LIMIT, + ); + setPreviewCache((prev) => ({ ...prev, [cacheKey]: parsed })); setViewModalData({ name, headers: parsed.headers, @@ -235,6 +281,13 @@ export default function DatasetsTab({ const file = event.dataTransfer.files?.[0]; if (!file || !isAllowedDatasetFile(file.name)) return; + if (file.size > MAX_DATASET_FILE_BYTES) { + toast.error( + `File too large. Max ${MAX_DATASET_FILE_BYTES / (1024 * 1024)}MB allowed.`, + ); + return; + } + const dataTransfer = new DataTransfer(); dataTransfer.items.add(file); if (!fileInputRef.current) return; diff --git a/app/components/assessment/datasets/CreatePanel.tsx b/app/components/assessment/datasets/CreatePanel.tsx index 976705f..d238b96 100644 --- a/app/components/assessment/datasets/CreatePanel.tsx +++ b/app/components/assessment/datasets/CreatePanel.tsx @@ -72,7 +72,7 @@ export default function CreatePanel({ />
{ + setSchema(draftSchema); + onClose(); + }; const handleReset = () => { setDraftSchema([createProperty()]); }; @@ -52,7 +56,7 @@ export default function OutputSchemaModal({ return createPortal(
{header} -
+
+
{cell || —}