Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { createContext, use, useCallback, useEffect, useMemo, useRef, useState, type ChangeEvent, type ReactNode } from 'react'
import { Database, FileCode, FileSpreadsheet, Loader2, Upload } from 'lucide-react'
import { isApolloError } from '@apollo/client'
import { Dialog, DialogClose, DialogContent, DialogFooter } from '@/components/ui/dialog'
import { Button } from '@/components/ui/Button'
import { Checkbox } from '@/components/ui/checkbox'
Expand Down Expand Up @@ -575,10 +576,12 @@ function DatabaseImportProvider({
await executeSqlImport()
}
} catch (error) {
const networkError = error instanceof Error && isApolloError(error) ? error.networkError : null
const requestTooLarge = method === 'sql' && networkError && 'statusCode' in networkError && networkError.statusCode === 413
actions.setAlert({
type: 'error',
title: t('database.import.failedTitle'),
message: error instanceof Error ? error.message : t(method === 'tableFile'
message: requestTooLarge ? t('database.import.error.requestTooLarge') : error instanceof Error ? error.message : t(method === 'tableFile'
? 'database.import.table.failedMessage'
: 'database.import.failedMessage'),
})
Expand Down
4 changes: 4 additions & 0 deletions dataflow/src/config/graphql-client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,10 @@ const uploadLink = new ApolloLink((operation) => new Observable((observer) => {
body: formData,
signal: controller.signal,
});
// A rejected upload can return plain text or HTML rather than GraphQL JSON.
if (response.status === 413) {
throw Object.assign(new Error('GraphQL upload failed (413)'), { statusCode: 413 });
}
const payload = await readGraphQLResponse(response);

if (!response.ok && !payload.errors) {
Expand Down
1 change: 1 addition & 0 deletions dataflow/src/i18n/locales/en/common.ts
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,7 @@ export const enCommonMessages = {
'database.import.error.sqlSourceBoth': 'Choose either a SQL file or SQL text, not both.',
'database.import.error.sqlSourceMissing': 'Provide a SQL file or SQL text.',
'database.import.error.sqlTooLarge': 'The SQL file exceeds the size limit.',
'database.import.error.requestTooLarge': 'The import content is too large. Reduce the SQL file or text size and try again.',
'database.import.error.sqlFileFailed': 'Failed to read the SQL file.',
'database.import.error.sqlMultiStatementUnsupported': 'This database does not support multi-statement SQL import.',
'database.import.error.sqlFailed':
Expand Down
1 change: 1 addition & 0 deletions dataflow/src/i18n/locales/zh/common.ts
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,7 @@ export const zhCommonMessages = {
'database.import.error.sqlSourceBoth': '只能选择 SQL 文件或 SQL 文本之一。',
'database.import.error.sqlSourceMissing': '请提供 SQL 文件或 SQL 文本。',
'database.import.error.sqlTooLarge': 'SQL 文件超过大小限制。',
'database.import.error.requestTooLarge': '导入内容过大,请减小 SQL 文件或文本后重试。',
'database.import.error.sqlFileFailed': 'SQL 文件读取失败。',
'database.import.error.sqlMultiStatementUnsupported': '当前数据库不支持多语句 SQL 导入。',
'database.import.error.sqlFailed': 'SQL 执行失败。请确认脚本语法与当前数据库匹配;例如 MySQL dump 不能直接导入 PostgreSQL。',
Expand Down
100 changes: 100 additions & 0 deletions dataflow/src/test/DatabaseImportModal.test.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
import { ApolloError } from '@apollo/client'
import { act, cleanup, fireEvent, render, screen, waitFor } from '@testing-library/react'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'

import { DatabaseImportModal } from '@/components/database/import/DatabaseImportModal'
import { I18nProvider } from '@/i18n/I18nProvider'
import { useConnectionStore, type Connection } from '@/stores/useConnectionStore'

const { importSql } = vi.hoisted(() => ({ importSql: vi.fn() }))

vi.mock('@graphql', async (importOriginal) => ({
...await importOriginal<typeof import('@graphql')>(),
useImportSqlMutation: () => [importSql],
useImportPreviewMutation: () => [vi.fn(), { loading: false }],
useImportTableFileMutation: () => [vi.fn()],
useGetDatabaseMetadataQuery: () => ({ loading: false }),
}))

const connection: Connection = {
id: 'postgres-1',
name: 'PostgreSQL @ localhost',
type: 'POSTGRES',
host: 'localhost',
port: '5432',
user: 'postgres',
password: '',
database: 'analytics',
createdAt: '2026-04-02T00:00:00.000Z',
}

const originalState = useConnectionStore.getState()
const script = 'SELECT 1;'
const tooLargeMessage = '导入内容过大,请减小 SQL 文件或文本后重试。'

async function prepareSqlImport(source: 'file' | 'text') {
render(
<I18nProvider locale="zh">
<DatabaseImportModal
open
onOpenChange={vi.fn()}
connectionId={connection.id}
databaseName="analytics"
/>
</I18nProvider>,
)
fireEvent.click(screen.getByRole('button', { name: /SQL 脚本/ }))

if (source === 'file') {
const file = new File([script], 'query.sql', { type: 'application/sql' })
// jsdom does not implement File.text().
Object.defineProperty(file, 'text', { value: async () => script })
fireEvent.change(screen.getByLabelText('上传 SQL 文件'), { target: { files: [file] } })
} else {
fireEvent.click(screen.getByRole('button', { name: '文本' }))
fireEvent.change(screen.getByLabelText('SQL 文本输入'), { target: { value: script } })
}

await waitFor(() => expect(screen.getByRole('button', { name: '执行导入' })).toBeEnabled())
await act(async () => {
fireEvent.click(screen.getByRole('button', { name: '执行导入' }))
})
}

describe('DatabaseImportModal SQL errors', () => {
beforeEach(() => {
importSql.mockReset()
useConnectionStore.setState({
...originalState,
connections: [connection],
fetchDatabases: vi.fn().mockResolvedValue(['analytics']),
})
})

afterEach(() => {
cleanup()
useConnectionStore.setState(originalState)
})

it.each(['file', 'text'] as const)('explains HTTP 413 for a SQL %s import', async (source) => {
importSql.mockRejectedValue(new ApolloError({
networkError: Object.assign(new Error('Received status code 413'), { statusCode: 413 }),
}))

await prepareSqlImport(source)

expect(await screen.findByText(tooLargeMessage)).toBeInTheDocument()
expect(screen.queryByText('Received status code 413')).not.toBeInTheDocument()
expect(importSql).toHaveBeenCalledOnce()
})

it('preserves ordinary SQL errors even when their message contains 413', async () => {
const message = 'SQL syntax error near column_413'
importSql.mockRejectedValue(new ApolloError({ errorMessage: message }))

await prepareSqlImport('text')

expect(await screen.findByText(message)).toBeInTheDocument()
expect(screen.queryByText(tooLargeMessage)).not.toBeInTheDocument()
})
})
33 changes: 33 additions & 0 deletions dataflow/src/test/graphql-client.test.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { gql } from '@apollo/client'
import { beforeEach, describe, expect, it, vi } from 'vitest'

const getAuthSessionMock = vi.fn()
Expand Down Expand Up @@ -101,3 +102,35 @@ describe('graphql client auth fetch', () => {
expect((form.get('0') as File).name).toBe('seed.sql')
})
})


describe('SQL import HTTP errors', () => {
beforeEach(() => {
vi.resetModules()
getAuthSessionMock.mockReturnValue(null)
vi.stubGlobal('fetch', vi.fn())
})

const mutation = gql`
mutation ImportSQL($input: ImportSQLInput!) {
ImportSQL(input: $input) { Status }
}
`

for (const kind of ['file', 'text']) {
it.each(['Request body too large', '<html>413 Request Entity Too Large</html>', ''])(
`${kind} imports preserve status 413 for response body %j`,
async (body) => {
vi.mocked(fetch).mockResolvedValue(new Response(body, { status: 413 }))
const { graphqlClient } = await import('@/config/graphql-client')
const input = kind === 'file'
? { File: new File(['SELECT 1;'], 'seed.sql'), Filename: 'seed.sql' }
: { Script: 'SELECT 1;' }

await expect(graphqlClient.mutate({ mutation, variables: { input } }))
.rejects.toMatchObject({ networkError: { statusCode: 413 } })
expect(fetch).toHaveBeenCalledTimes(1)
},
)
}
})
27 changes: 21 additions & 6 deletions docs/runbook.md
Original file line number Diff line number Diff line change
Expand Up @@ -111,23 +111,38 @@ Delete generated binaries after local testing unless they are intentional artifa

## Build Runtime Image

Build an `amd64` image by default for release-oriented checks:
Build and publish both `amd64` and `arm64` images by default:

```bash
docker buildx build \
-f core/Dockerfile \
--platform linux/amd64 \
--platform linux/amd64,linux/arm64 \
--build-arg VERSION=<version> \
--build-arg TARGETARCH=amd64 \
--build-arg PLATFORM=docker \
-t dataflow-local:<version> \
.
-t <registry>/dataflow:<version> \
--push .
```

Buildx supplies `TARGETARCH` for each platform; do not pin it to one architecture
in a multi-platform build. For test deployments, use
`crpi-7jr40k6elhldekqp.cn-hangzhou.personal.cr.aliyuncs.com/mlhiter` unless another
registry is explicitly requested.

If the registry rejects the build with `unknown manifest class for
application/vnd.oci.empty.v1+json`, rebuild with `--provenance=false --sbom=false`
and verify that the published index contains both target platforms. The existing
build layers can be reused.

If the Go compiler is killed while compiling Elasticsearch packages on a
memory-constrained builder, use a temporary Dockerfile with
`ENV GOGC=20 GOFLAGS=-p=1 GOMAXPROCS=2` immediately before the backend `go build`
step. This reduces compilation concurrency and memory pressure at the cost of
build time; keep these settings out of the final runtime stage.

Run locally:

```bash
docker run --rm -p 8080:8080 dataflow-local:<version>
docker run --rm -p 8080:8080 <registry>/dataflow:<version>
```

Open `http://localhost:8080`.
Expand Down
Loading