Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -7,18 +7,51 @@ export type BigQueryValidationIssue = {
message: string
}

const BIGQUERY_REQUIRED_FIELDS: { path: BigQueryFieldPath; message: string }[] = [
export const BIGQUERY_SERVICE_ACCOUNT_JSON_MESSAGE = 'Service account key must be valid JSON'

const BIGQUERY_REQUIRED_FIELDS: {
path: Exclude<BigQueryFieldPath, 'serviceAccountKey'>
message: string
}[] = [
{ path: 'projectId', message: 'Project ID is required' },
{ path: 'datasetId', message: 'Dataset ID is required' },
{ path: 'serviceAccountKey', message: 'Service account key is required' },
]

const isValidJsonString = (value: string) => {
try {
JSON.parse(value)
return true
} catch {
return false
}
}

export const getBigQueryValidationIssues = (
data: Pick<DestinationPanelSchemaType, BigQueryFieldPath>,
options: { secretsOptional?: boolean } = {}
): BigQueryValidationIssue[] =>
BIGQUERY_REQUIRED_FIELDS.filter(({ path }) => {
if (options.secretsOptional && path === 'serviceAccountKey') return false
options: { secretsOptional?: boolean; validateJson?: boolean } = {}
): BigQueryValidationIssue[] => {
const { secretsOptional = false, validateJson = true } = options
const issues: BigQueryValidationIssue[] = BIGQUERY_REQUIRED_FIELDS.filter(
({ path }) => !data[path]?.trim().length
).map(({ path, message }) => ({ path, message }))

return !data[path]?.trim().length
})
const serviceAccountKey = data.serviceAccountKey?.trim() ?? ''

if (!serviceAccountKey) {
if (!secretsOptional) {
issues.push({ path: 'serviceAccountKey', message: 'Service account key is required' })
}
return issues
}

// JSON shape is checked on submit only. Live onChange validation would fail on every
// keystroke while the user is still pasting or typing a key.
if (validateJson && !isValidJsonString(serviceAccountKey)) {
issues.push({
path: 'serviceAccountKey',
message: BIGQUERY_SERVICE_ACCOUNT_JSON_MESSAGE,
})
}

return issues
}
Original file line number Diff line number Diff line change
Expand Up @@ -769,6 +769,58 @@ describe('DestinationForm.utils BigQuery', () => {
expect(issues).toEqual([])
})

it('rejects invalid JSON in the service account key', () => {
const issues = getBigQueryValidationIssues({
projectId: 'my-project',
datasetId: 'my_dataset',
serviceAccountKey: '{ "type": "service_account" ',
})

expect(issues).toEqual([
{ path: 'serviceAccountKey', message: 'Service account key must be valid JSON' },
])
})

it('skips JSON shape checks when validateJson is false', () => {
const issues = getBigQueryValidationIssues(
{
projectId: 'my-project',
datasetId: 'my_dataset',
serviceAccountKey: '{ "type": "service_account" ',
},
{ validateJson: false }
)

expect(issues).toEqual([])
})

it('rejects non-JSON text in the service account key', () => {
const issues = getBigQueryValidationIssues({
projectId: 'my-project',
datasetId: 'my_dataset',
serviceAccountKey: 'not-json',
})

expect(issues).toEqual([
{ path: 'serviceAccountKey', message: 'Service account key must be valid JSON' },
])
})

it('validates JSON when replacing credentials in edit mode', () => {
const issues = getBigQueryValidationIssues(
{
projectId: 'my-project',
datasetId: 'my_dataset',
serviceAccountKey: '{ invalid',
},
{ secretsOptional: true }
)

expect(issues).toEqual([
{ path: 'serviceAccountKey', message: 'Service account key must be valid JSON' },
])
})

it('allows an omitted BigQuery service account key in edit mode', () => {
const issues = getBigQueryValidationIssues(
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,15 @@ export const DestinationNameInput = ({ form }: DestinationNameInputProps) => {
render={({ field }) => (
<FormItemLayout label="Name" layout="horizontal">
<FormControl>
<Input {...field} placeholder="My destination" />
<Input
{...field}
autoFocus
placeholder="My destination"
data-1p-ignore
data-lpignore="true"
data-form-type="other"
data-bwignore
/>
</FormControl>
</FormItemLayout>
)}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
import { screen } from '@testing-library/react'
import { describe, expect, test } from 'vitest'

import {
getPipelineRegionDescription,
PIPELINE_REGION,
PipelineRegionField,
} from './PipelineRegionField'
import { customRender } from '@/tests/lib/custom-render'

describe('getPipelineRegionDescription', () => {
test('includes the fixed-region note and destination-specific hint', () => {
expect(getPipelineRegionDescription('BigQuery')).toBe(
'All pipelines run from this region. Choose a nearby BigQuery dataset where possible.'
)
})

test('falls back to a generic destination hint', () => {
expect(getPipelineRegionDescription()).toBe(
'All pipelines run from this region. Choose a nearby destination region where possible.'
)
})
})

describe('PipelineRegionField', () => {
test('shows the managed region as read-only information, not a combobox', () => {
customRender(<PipelineRegionField destinationType="BigQuery" />)

expect(screen.getByText('Pipeline region')).toBeInTheDocument()
expect(screen.getByText(PIPELINE_REGION.displayName)).toBeInTheDocument()
expect(screen.getByText(PIPELINE_REGION.code)).toBeInTheDocument()
expect(screen.getByText(/All pipelines run from this region/)).toBeInTheDocument()
expect(screen.getByText(/nearby BigQuery dataset/)).toBeInTheDocument()
expect(screen.queryByRole('combobox')).not.toBeInTheDocument()
})
})
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
import { AWS_REGIONS } from 'shared-data'
import { cn } from 'ui'
import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout'

import type { DestinationType } from '../DestinationPanel.types'
import { RegionFlag } from '@/components/ui/RegionFlag'
import { IS_STAGING_OR_LOCAL } from '@/lib/constants'

// Pipelines always run from a single fixed region per environment, regardless of the source
// project's region.
export const PIPELINE_REGION = IS_STAGING_OR_LOCAL
? AWS_REGIONS.SOUTHEAST_ASIA
: AWS_REGIONS.CENTRAL_EU

const DESTINATION_REGION_HINT: Record<DestinationType, string> = {
BigQuery: 'Choose a nearby BigQuery dataset where possible.',
'Analytics Bucket': 'Keep this bucket in the same region where possible.',
DuckLake: 'Keep catalog and object storage close to this region where possible.',
Snowflake: 'Choose a nearby Snowflake account region where possible.',
ClickHouse: 'Choose a nearby ClickHouse cluster where possible.',
}

export const getPipelineRegionDescription = (destinationType?: DestinationType) => {
const destinationHint = destinationType
? DESTINATION_REGION_HINT[destinationType]
: 'Choose a nearby destination region where possible.'

return `All pipelines run from this region. ${destinationHint}`
}

export const PipelineRegionField = ({
destinationType,
className,
}: {
destinationType?: DestinationType
className?: string
}) => {
return (
<FormItemLayout
isReactForm={false}
layout="horizontal"
label="Pipeline region"
description={getPipelineRegionDescription(destinationType)}
>
<div
className={cn(
'flex h-9 min-w-0 items-center gap-x-2 rounded-md border bg-surface-200 px-3 text-sm',
className
)}
>
<RegionFlag className="w-5 shrink-0" region={PIPELINE_REGION.code} />
<span className="min-w-0 truncate text-foreground">{PIPELINE_REGION.displayName}</span>
<span className="shrink-0 font-mono text-xs text-foreground-lighter">
{PIPELINE_REGION.code}
</span>
</div>
</FormItemLayout>
)
}
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ vi.mock('ui', () => ({
{children}
</button>
),
cn: (...args: unknown[]) => args.filter(Boolean).join(' '),
DialogSectionSeparator: () => null,
Form: ({ children }: PropsWithChildren) => children,
Select: ({ children }: PropsWithChildren) => <div>{children}</div>,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,25 +5,9 @@ import { AnimatePresence, motion } from 'framer-motion'
import { Loader2 } from 'lucide-react'
import { RefObject, useEffect, useMemo, useRef, useState, type ReactNode } from 'react'
import { useForm, useWatch } from 'react-hook-form'
import { AWS_REGIONS } from 'shared-data'
import { toast } from 'sonner'
import {
Button,
DialogSectionSeparator,
Form,
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
SheetFooter,
SheetSection,
Tooltip,
TooltipContent,
TooltipTrigger,
} from 'ui'
import { Button, DialogSectionSeparator, Form, SheetFooter, SheetSection } from 'ui'
import { Admonition } from 'ui-patterns/Admonition'
import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout'
import * as z from 'zod'

import {
Expand All @@ -37,7 +21,10 @@ import { type DestinationType, type ExistingDestination } from '../DestinationPa
import { AdvancedSettings } from './AdvancedSettings'
import { getAnalyticsBucketValidationIssues } from './AnalyticsBucket/AnalyticsBucket.utils'
import { AnalyticsBucketFields } from './AnalyticsBucket/Fields'
import { getBigQueryValidationIssues } from './BigQuery/BigQuery.utils'
import {
BIGQUERY_SERVICE_ACCOUNT_JSON_MESSAGE,
getBigQueryValidationIssues,
} from './BigQuery/BigQuery.utils'
import { BigQueryFields } from './BigQuery/Fields'
import { getClickHouseValidationIssues } from './ClickHouse/ClickHouse.utils'
import { ClickHouseFields } from './ClickHouse/Fields'
Expand All @@ -54,6 +41,7 @@ import { DuckLakeFields } from './DuckLake/Fields'
import { NewPublicationPanel } from './NewPublicationPanel'
import { NoDestinationsAvailable } from './NoDestinationsAvailable'
import { PipelineCostDialog } from './PipelineCostDialog'
import { PipelineRegionField } from './PipelineRegionField'
import { PublicationSelection } from './PublicationSelection'
import { SnowflakeFields } from './Snowflake/Fields'
import { getSnowflakeValidationIssues } from './Snowflake/Snowflake.utils'
Expand All @@ -62,23 +50,16 @@ import { useDestinationForm } from './useDestinationForm'
import { ValidationFailuresSection } from './ValidationFailuresSection'
import { ValidationWarningsDialog } from './ValidationWarningsDialog'
import { CreateAnalyticsBucketSheet } from '@/components/interfaces/Storage/AnalyticsBuckets/CreateAnalyticsBucketSheet'
import { InlineLinkClassName } from '@/components/ui/InlineLink'
import { RegionFlag } from '@/components/ui/RegionFlag'
import { useAPIKeys } from '@/data/api-keys/api-keys-query'
import { useProjectSettingsV2Query } from '@/data/config/project-settings-v2-query'
import { useReplicationDestinationByIdQuery } from '@/data/replication/destination-by-id-query'
import { useReplicationPipelineByIdQuery } from '@/data/replication/pipeline-by-id-query'
import { useReplicationPublicationsQuery } from '@/data/replication/publications-query'
import { useReplicationSourceId } from '@/data/replication/sources-query'
import { useAsyncCheckPermissions } from '@/hooks/misc/useCheckPermissions'
import { IS_STAGING_OR_LOCAL } from '@/lib/constants'

const formId = 'destination-editor'

// Pipelines always run out of a single fixed region per environment, regardless of the source
// project's region.
const PIPELINE_REGION = IS_STAGING_OR_LOCAL ? AWS_REGIONS.SOUTHEAST_ASIA : AWS_REGIONS.CENTRAL_EU

interface DestinationFormProps {
selectedType: DestinationType
visible: boolean
Expand Down Expand Up @@ -232,11 +213,12 @@ export const DestinationForm = ({
}

if (selectedType === 'BigQuery') {
getBigQueryValidationIssues(data, { secretsOptional: editMode }).forEach(
({ path, message }) => {
addRequiredFieldError(path, message)
}
)
getBigQueryValidationIssues(data, {
secretsOptional: editMode,
validateJson: false,
}).forEach(({ path, message }) => {
addRequiredFieldError(path, message)
})
} else if (selectedType === 'Analytics Bucket') {
getAnalyticsBucketValidationIssues(data, {
secretsOptional: editMode,
Expand Down Expand Up @@ -342,6 +324,16 @@ export const DestinationForm = ({
}),
}

if (selectedType === 'BigQuery') {
const jsonIssue = getBigQueryValidationIssues(data, { secretsOptional: editMode }).find(
(issue) => issue.message === BIGQUERY_SERVICE_ACCOUNT_JSON_MESSAGE
)
if (jsonIssue) {
form.setError(jsonIssue.path, { message: jsonIssue.message })
return
}
}

// Pipeline prerequisite validation models a new pipeline and cannot
// account for resources already owned by an existing pipeline. Edits keep
// the established direct-update flow after pruning stale table ids.
Expand Down Expand Up @@ -469,42 +461,7 @@ export const DestinationForm = ({
onSelectNewPublication={() => setPublicationPanelVisible(true)}
/>
<TableCopySelection form={form} editMode={editMode} />
<FormItemLayout
isReactForm={false}
layout="horizontal"
label="Region"
description={
<span className="text-foreground-lighter">
Pipelines run in{' '}
<Tooltip>
<TooltipTrigger className={InlineLinkClassName}>
{PIPELINE_REGION.displayName}
</TooltipTrigger>
<TooltipContent side="bottom">{PIPELINE_REGION.code}</TooltipContent>
</Tooltip>
. In your destination provider, choose the closest available region.
</span>
}
>
<Select disabled value={PIPELINE_REGION.code}>
<SelectTrigger>
<SelectValue placeholder="Select a region" />
</SelectTrigger>
<SelectContent>
<SelectItem value={PIPELINE_REGION.code}>
<div className="flex gap-x-3 items-center">
<RegionFlag className="w-5" region={PIPELINE_REGION.code} />
<p className="flex items-center gap-x-2">
<span>{PIPELINE_REGION.displayName}</span>
<span className="text-xs text-foreground-lighter font-mono">
{PIPELINE_REGION.code}
</span>
</p>
</div>
</SelectItem>
</SelectContent>
</Select>
</FormItemLayout>
<PipelineRegionField destinationType={selectedType} />
</div>
</div>

Expand Down
Loading
Loading