diff --git a/.changeset/ledger.yaml b/.changeset/ledger.yaml index 23aeaa0d5e23..7fb4f25bb27c 100644 --- a/.changeset/ledger.yaml +++ b/.changeset/ledger.yaml @@ -1,124 +1,124 @@ -"@tryghost/adapter-base-cache@0.2.0": +'@tryghost/adapter-base-cache@0.2.0': dir: packages/adapters/cache-base intents: - major-areas-fail - upset-dogs-sort -"@tryghost/adapter-base-scheduling@0.1.1": +'@tryghost/adapter-base-scheduling@0.1.1': dir: packages/adapters/scheduling-base intents: - major-areas-fail - plain-singers-cheat -"@tryghost/adapter-base-scheduling@0.2.0": +'@tryghost/adapter-base-scheduling@0.2.0': dir: packages/adapters/scheduling-base intents: - fifty-maps-sing -"@tryghost/adapter-base-scheduling@0.2.1": +'@tryghost/adapter-base-scheduling@0.2.1': dir: packages/adapters/scheduling-base intents: - typed-logging-args -"@tryghost/adapter-base-scheduling@0.2.2": +'@tryghost/adapter-base-scheduling@0.2.2': dir: packages/adapters/scheduling-base intents: - open-shirts-arrive - young-spiders-request -"@tryghost/adapter-base-scheduling@0.2.3": +'@tryghost/adapter-base-scheduling@0.2.3': dir: packages/adapters/scheduling-base intents: - brown-rules-cut -"@tryghost/adapter-base-sso@0.1.1": +'@tryghost/adapter-base-sso@0.1.1': dir: packages/adapters/sso-base intents: - major-areas-fail - plain-singers-cheat -"@tryghost/adapter-base-sso@0.1.2": +'@tryghost/adapter-base-sso@0.1.2': dir: packages/adapters/sso-base intents: - open-shirts-arrive -"@tryghost/adapter-base-sso@0.1.3": +'@tryghost/adapter-base-sso@0.1.3': dir: packages/adapters/sso-base intents: - brown-rules-cut -"@tryghost/kg-card-factory@5.2.4": +'@tryghost/kg-card-factory@5.2.4': dir: koenig/kg-card-factory intents: - common-squids-argue -"@tryghost/kg-clean-basic-html@4.3.4": +'@tryghost/kg-clean-basic-html@4.3.4': dir: koenig/kg-clean-basic-html intents: - common-squids-argue - six-parents-shout -"@tryghost/kg-converters@1.2.4": +'@tryghost/kg-converters@1.2.4': dir: koenig/kg-converters intents: - tidy-types-check -"@tryghost/kg-converters@1.2.5": +'@tryghost/kg-converters@1.2.5': dir: koenig/kg-converters intents: - common-squids-argue -"@tryghost/kg-default-cards@10.3.4": +'@tryghost/kg-default-cards@10.3.4': dir: koenig/kg-default-cards intents: - spacy-poodles-hunt -"@tryghost/kg-default-cards@10.3.5": +'@tryghost/kg-default-cards@10.3.5': dir: koenig/kg-default-cards intents: - common-squids-argue - six-parents-shout -"@tryghost/kg-default-nodes@2.1.4": +'@tryghost/kg-default-nodes@2.1.4': dir: koenig/kg-default-nodes intents: - floppy-bobcats-spend -"@tryghost/kg-default-nodes@2.1.5": +'@tryghost/kg-default-nodes@2.1.5': dir: koenig/kg-default-nodes intents: - spacy-poodles-hunt - warm-hotels-make -"@tryghost/kg-default-nodes@2.2.0": +'@tryghost/kg-default-nodes@2.2.0': dir: koenig/kg-default-nodes intents: - clean-flags-retire - common-squids-argue - six-parents-shout -"@tryghost/kg-default-transforms@1.3.4": +'@tryghost/kg-default-transforms@1.3.4': dir: koenig/kg-default-transforms intents: - common-squids-argue -"@tryghost/kg-html-to-lexical@1.4.0": +'@tryghost/kg-html-to-lexical@1.4.0': dir: koenig/kg-html-to-lexical intents: - common-squids-argue - six-parents-shout -"@tryghost/kg-lexical-html-renderer@1.5.0": +'@tryghost/kg-lexical-html-renderer@1.5.0': dir: koenig/kg-lexical-html-renderer intents: - common-squids-argue - six-parents-shout -"@tryghost/kg-markdown-html-renderer@7.2.4": +'@tryghost/kg-markdown-html-renderer@7.2.4': dir: koenig/kg-markdown-html-renderer intents: - common-squids-argue - four-rings-relate -"@tryghost/kg-unsplash-selector@0.4.4": +'@tryghost/kg-unsplash-selector@0.4.4': dir: koenig/kg-unsplash-selector intents: - common-squids-argue - petite-schools-accept - six-parents-shout - swift-guests-grin -"@tryghost/kg-utils@1.1.4": +'@tryghost/kg-utils@1.1.4': dir: koenig/kg-utils intents: - common-squids-argue - four-rings-relate -"@tryghost/koenig-lexical@1.9.0": +'@tryghost/koenig-lexical@1.9.0': dir: koenig/koenig-lexical intents: - bright-walls-preview -"@tryghost/koenig-lexical@1.9.1": +'@tryghost/koenig-lexical@1.9.1': dir: koenig/koenig-lexical intents: - warm-hotels-make -"@tryghost/koenig-lexical@1.9.2": +'@tryghost/koenig-lexical@1.9.2': dir: koenig/koenig-lexical intents: - funky-bikes-chew @@ -128,7 +128,7 @@ - petite-forks-lick - petite-schools-accept - six-parents-shout -"@tryghost/koenig-lexical@1.9.3": +'@tryghost/koenig-lexical@1.9.3': dir: koenig/koenig-lexical intents: - plenty-moons-smile diff --git a/.github/renovate.json5 b/.github/renovate.json5 index 53518379cd1c..3af2791d3c2a 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -370,7 +370,6 @@ '@tryghost/mw-vhost', '@tryghost/pretty-cli', '@tryghost/prometheus-metrics', - '@tryghost/promise', '@tryghost/referrer-parser', '@tryghost/root-utils', '@tryghost/security', diff --git a/apps/admin-x-framework/test/unit/api/member-custom-fields.test.ts b/apps/admin-x-framework/test/unit/api/member-custom-fields.test.ts index ebb9a85ab022..4c1a21db5784 100644 --- a/apps/admin-x-framework/test/unit/api/member-custom-fields.test.ts +++ b/apps/admin-x-framework/test/unit/api/member-custom-fields.test.ts @@ -7,8 +7,8 @@ import { } from '../../../src/api/member-custom-fields'; // Compile-time cases: the build failing is the assertion. Each `@ts-expect-error` fails the -// build if the case it names stops being an error. Declared on one line each, because the -// directive only covers the line below it and a spread literal reports on its inner line. +// build if the case it names stops being an error. Directives sit immediately above the +// property where TypeScript reports the error. const composite = { line1: 'a', line2: 'b', city: 'c', state: 'd', postal_code: 'e', country: 'f' }; const labelled: FieldTypePresentation<'address'> = { @@ -17,27 +17,27 @@ const labelled: FieldTypePresentation<'address'> = { subFields: composite, }; -// @ts-expect-error a composite missing one of the parts its value schema declares const missingPart: FieldTypePresentation<'address'> = { label: 'Address', input: 'address', + // @ts-expect-error a composite missing one of the parts its value schema declares subFields: { line1: 'a', line2: 'b', city: 'c', state: 'd', country: 'f' }, }; -// @ts-expect-error a composite naming a part its value schema does not declare const unknownPart: FieldTypePresentation<'address'> = { label: 'Address', input: 'address', + // @ts-expect-error a composite naming a part its value schema does not declare subFields: { ...composite, county: 'g' }, }; // @ts-expect-error a composite with no part labels at all const unlabelled: FieldTypePresentation<'address'> = { label: 'Address', input: 'address' }; -// @ts-expect-error a type whose value is one thing has no parts to name const scalarWithParts: FieldTypePresentation<'short_text'> = { label: 'Short text', input: 'text', + // @ts-expect-error a type whose value is one thing has no parts to name subFields: { line1: 'a' }, }; diff --git a/apps/admin-x-framework/test/unit/api/posts.test.tsx b/apps/admin-x-framework/test/unit/api/posts.test.tsx index daa03a4491e7..dbe4ffd35cf8 100644 --- a/apps/admin-x-framework/test/unit/api/posts.test.tsx +++ b/apps/admin-x-framework/test/unit/api/posts.test.tsx @@ -55,4 +55,20 @@ describe('posts api', () => { vi.useRealTimers(); } }); + + it('uploads a mapped CSV ZIP through the posts endpoint', async () => { + const file = new File(['PK'], 'posts.zip', { type: 'application/zip' }); + + await withMockFetch({}, async (mock) => { + const { result } = renderHookWithProviders(() => useImportContentCSV()); + + await act(async () => { + await result.current.mutateAsync({ file, mapping: { Headline: 'title' } }); + }); + + expect(mock.calls[0][0]).toBe('http://localhost:3000/ghost/api/admin/posts/upload/'); + expect(mock.calls[0][1].body.get('postsfile')).toBe(file); + expect(mock.calls[0][1].body.get('mapping[Headline]')).toBe('title'); + }); + }); }); diff --git a/apps/admin/src/members/components/bulk-action-modals/import-members/csv.test.ts b/apps/admin/src/members/components/bulk-action-modals/import-members/csv.test.ts index cb879bf58397..5c3c39dc8ed4 100644 --- a/apps/admin/src/members/components/bulk-action-modals/import-members/csv.test.ts +++ b/apps/admin/src/members/components/bulk-action-modals/import-members/csv.test.ts @@ -32,16 +32,97 @@ describe('csv helpers', () => { ]); }); - it('serializes error rows to a downloadable CSV string', () => { + it('serializes error rows into the members export format', () => { const output = unparseErrorCSV([ { name: 'Alice', email: 'bad@example.com', + subscribed: true, + complimentary_plan: false, + labels: [{ name: 'vip' }, { name: 'gold' }], error: 'Invalid email, "quote"', }, ]); - expect(output).toContain('"name","email","error"'); - expect(output).toContain('"Alice","bad@example.com","Invalid email, ""quote"""'); + expect(output).toContain( + '"email","name","note","subscribed_to_emails","complimentary_plan","stripe_customer_id","created_at","labels","gift_id","error"', + ); + expect(output).toContain('"vip,gold"'); + expect(output.split('\n')[1]).toContain('"true"'); + expect(output).not.toContain('[object Object]'); + expect(output).toContain('"Invalid email, ""quote"""'); + }); + + it('does not invent subscription columns when the source omitted them', () => { + const output = unparseErrorCSV([ + { + email: 'a@example.com', + labels: [], + error: 'nope', + }, + ]); + + const [header] = output.split('\r\n'); + expect(header).not.toContain('subscribed_to_emails'); + expect(header).not.toContain('complimentary_plan'); + }); + + it('drops malformed label entries instead of crashing the download', () => { + const output = unparseErrorCSV([ + { + email: 'a@example.com', + labels: [null, { name: 'vip' }, 42, { name: 'gold' }], + error: 'nope', + }, + ]); + + expect(output).toContain('"vip,gold"'); + expect(output).not.toContain('[object Object]'); + }); + + it('escapes formula-like values before writing the downloadable CSV', () => { + const output = unparseErrorCSV([ + { + name: '=1+2', + email: 'a@example.com', + error: 'nope', + }, + ]); + + expect(output).toContain(`"'=1+2"`); + }); + + it('keeps a column carried only by a later row, with the error column last', () => { + const output = unparseErrorCSV([ + { email: 'a@example.com', labels: [], error: 'nope' }, + { + email: 'b@example.com', + labels: [], + newsletters: [{ name: 'Daily News' }], + 'custom_fields.topic': 'ghosts', + error: 'nope', + }, + ]); + + const header = output.split('\n')[0].trimEnd(); + expect(header).toContain('"newsletters"'); + expect(header).toContain('"custom_fields.topic"'); + expect(header.endsWith('"error"')).toBe(true); + }); + + it('carries the newsletters column only when the submitted file did', () => { + const withColumn = unparseErrorCSV([ + { + email: 'a@example.com', + labels: [], + newsletters: [{ name: 'Daily News' }], + error: 'nope', + }, + ]); + expect(withColumn).toContain('"newsletters"'); + expect(withColumn).toContain('"Daily News"'); + + const withoutColumn = unparseErrorCSV([{ email: 'a@example.com', labels: [], error: 'nope' }]); + expect(withoutColumn.split('\n')[0]).not.toContain('newsletters'); }); }); diff --git a/apps/admin/src/members/components/bulk-action-modals/import-members/csv.ts b/apps/admin/src/members/components/bulk-action-modals/import-members/csv.ts index 1d996bd9a873..8ab198023930 100644 --- a/apps/admin/src/members/components/bulk-action-modals/import-members/csv.ts +++ b/apps/admin/src/members/components/bulk-action-modals/import-members/csv.ts @@ -1,4 +1,5 @@ import Papa from 'papaparse'; +import { z } from 'zod'; export function parseCSV(text: string): Record[] { const parsed = Papa.parse(text, { @@ -24,12 +25,103 @@ export function parseCSV(text: string): Record[] { }); } -export function unparseErrorCSV(rows: Array & { error: string }>): string { +// The API hands back the raw import rows, where labels and newsletters are arrays of +// {name} objects and field names are internal (subscribed, not subscribed_to_emails). +// Downloaded as-is they render as [object Object] and cannot be fixed and re-uploaded, +// so rows are shaped into the members export vocabulary first, matching the emailed +// error report. The newsletters column only appears when the submitted file carried +// one: an empty cell in that column reads back as an explicit "no newsletters". +const nameEntrySchema = z.union([z.string(), z.object({ name: z.string() })]); + +const rawErrorRowSchema = z.object({ error: z.string().catch('') }).catchall(z.unknown()); + +type RawErrorRow = z.infer; + +function joinNames(value: unknown): string { + if (typeof value === 'string') { + return value; + } + if (!Array.isArray(value)) { + return ''; + } + return value + .map((item) => { + const parsed = nameEntrySchema.safeParse(item); + if (!parsed.success) { + return ''; + } + return typeof parsed.data === 'string' ? parsed.data : parsed.data.name; + }) + .filter(Boolean) + .join(','); +} + +function cell(value: unknown): string { + if (typeof value === 'string') { + return value; + } + if (typeof value === 'number' || typeof value === 'boolean') { + return String(value); + } + return ''; +} + +function toExportErrorRow(row: RawErrorRow): Record { + const shaped: Record = { + email: cell(row.email), + name: cell(row.name), + note: cell(row.note), + ...(row.subscribed !== undefined ? { subscribed_to_emails: cell(row.subscribed) } : {}), + ...(row.complimentary_plan !== undefined + ? { complimentary_plan: cell(row.complimentary_plan) } + : {}), + stripe_customer_id: cell(row.stripe_customer_id), + created_at: cell(row.created_at), + labels: joinNames(row.labels), + }; + + // An absent subscription column must stay absent. The importer treats a present + // blank subscribed_to_emails cell as true, so inventing this column would change + // subscription state when the error report is re-uploaded. + if (row.newsletters !== undefined) { + shaped.newsletters = joinNames(row.newsletters); + } + + for (const [key, value] of Object.entries(row)) { + if (key.startsWith('custom_fields.')) { + shaped[key] = cell(value); + } + } + + shaped.gift_id = cell(row.gift_id); + shaped.error = cell(row.error); + + return shaped; +} + +export function unparseErrorCSV(rows: unknown[]): string { if (rows.length === 0) { return ''; } - return Papa.unparse(rows, { + const shaped = rows.map((row) => toExportErrorRow(rawErrorRowSchema.parse(row))); + + // Columns come from every row, not just the first: Papa.unparse otherwise takes the + // header from the first row's keys, dropping an optional column (newsletters, a + // custom field) that only a later row carries. The error column stays last. + const columns: string[] = []; + for (const row of shaped) { + for (const key of Object.keys(row)) { + if (key !== 'error' && !columns.includes(key)) { + columns.push(key); + } + } + } + columns.push('error'); + + return Papa.unparse(shaped, { quotes: true, + escapeFormulae: true, + columns, }); } diff --git a/apps/admin/src/members/components/bulk-action-modals/import-members/upload.test.ts b/apps/admin/src/members/components/bulk-action-modals/import-members/upload.test.ts index 1647f6c3fd1e..f44e459fdfc1 100644 --- a/apps/admin/src/members/components/bulk-action-modals/import-members/upload.test.ts +++ b/apps/admin/src/members/components/bulk-action-modals/import-members/upload.test.ts @@ -215,6 +215,29 @@ describe('buildImportResponse', () => { expect(result.errorList).toEqual([]); }); + it('handles malformed invalid rows without crashing the error download', async () => { + const result = buildImportResponse({ + meta: { + stats: { + imported: 0, + invalid: [ + null as never, + { + email: 'bad@example.com', + errors: 'not-an-array' as never, + error: 'bad row', + }, + ], + }, + }, + }); + + const csv = await (mockCreateObjectURL.mock.calls[0][0] as Blob).text(); + expect(result.errorCount).toBe(2); + expect(result.errorList).toEqual([]); + expect(csv).toContain('"error"'); + }); + it('creates a downloadable blob URL for the error CSV', () => { buildImportResponse({ meta: { @@ -237,8 +260,9 @@ describe('buildImportResponse', () => { expect(blob.type).toBe('text/csv'); }); - it('writes the error file with the submitted columns and no others', async () => { - // The file echoes the row back, so every key on it becomes a column. + it('writes the error file with export columns and submitted custom fields', async () => { + // The file uses the members export vocabulary, then carries through custom fields + // from the submitted row so the failed data can be fixed and re-uploaded. buildImportResponse({ meta: { stats: { @@ -264,7 +288,10 @@ describe('buildImportResponse', () => { // CRLF between rows; a reason may itself contain a bare newline. const [header] = csv.split('\r\n'); - expect(header).toBe('"email","custom_fields.home_address.country","error"'); + expect(header).toBe( + '"email","name","note","stripe_customer_id","created_at","labels","custom_fields.home_address.country","gift_id","error"', + ); + expect(csv).toContain('"IRL"'); expect(csv).toContain( '"Missing email address\ncustom_fields.home_address.country: Enter a 2-letter country code, like US."', ); diff --git a/apps/admin/src/members/components/bulk-action-modals/import-members/upload.ts b/apps/admin/src/members/components/bulk-action-modals/import-members/upload.ts index 6cb509ad3bb2..fac30d1257b5 100644 --- a/apps/admin/src/members/components/bulk-action-modals/import-members/upload.ts +++ b/apps/admin/src/members/components/bulk-action-modals/import-members/upload.ts @@ -1,16 +1,27 @@ import moment from 'moment-timezone'; +import { z } from 'zod'; import { type ImportMembersCompleteResponseType } from '@tryghost/admin-x-framework/api/members'; import { type ImportResponse } from './state'; import { formatImportError } from './mapping'; import { unparseErrorCSV } from './csv'; +const invalidImportRowSchema = z + .object({ + errors: z.array(z.string()).catch([]), + }) + .catchall(z.unknown()); + export function buildImportResponse(importData: ImportMembersCompleteResponseType): ImportResponse { const importedCount = importData.meta.stats.imported; - const erroredMembers = importData.meta.stats.invalid || []; + const erroredMembers = Array.isArray(importData.meta.stats.invalid) + ? importData.meta.stats.invalid + : []; const errorCount = erroredMembers.length; const errorListMap: Record = {}; - const errorsWithFormattedMessages = erroredMembers.map((row) => { + const errorsWithFormattedMessages = erroredMembers.map((rawRow) => { + const parsedRow = invalidImportRowSchema.safeParse(rawRow); + const row = parsedRow.success ? parsedRow.data : { errors: [] }; const { errors, ...columns } = row; const formatted = errors.map((reason) => formatImportError(reason).trim()).filter(Boolean); for (const reason of formatted) { diff --git a/apps/admin/src/settings/advanced/migration-tools/content-import/archive.test.ts b/apps/admin/src/settings/advanced/migration-tools/content-import/archive.test.ts new file mode 100644 index 000000000000..6ff239a4681b --- /dev/null +++ b/apps/admin/src/settings/advanced/migration-tools/content-import/archive.test.ts @@ -0,0 +1,136 @@ +import JSZip from 'jszip'; +import { inspectImportArchive } from './archive'; + +async function zipFile(name: string, build: (archive: JSZip) => void): Promise { + const archive = new JSZip(); + build(archive); + const bytes = await archive.generateAsync({ type: 'arraybuffer' }); + return new File([bytes], name, { type: 'application/zip' }); +} + +describe('content import archive', () => { + it('reads one root CSV without inflating asset files', async () => { + const file = await zipFile('posts.zip', (archive) => { + archive.file('posts.CSV', 'title\nHello'); + archive.file('content/images/large.jpg', new Uint8Array([1, 2, 3])); + archive.file('content/files/attachment.csv', 'download,only'); + archive.file('content/files/attachment.json', '{"download":true}'); + archive.file('content/files/attachment.md', '# Download'); + }); + + await expect(inspectImportArchive(file)).resolves.toEqual({ + type: 'csv', + csv: 'title\nHello', + name: 'posts.CSV', + }); + }); + + it('reads one CSV inside a wrapper directory', async () => { + const file = await zipFile('posts.zip', (archive) => { + archive.file('export/posts.csv', 'title\nWrapped'); + archive.file('export/content/files/attachment.csv', 'download,only'); + archive.file('.DS_Store', 'metadata'); + }); + + await expect(inspectImportArchive(file)).resolves.toMatchObject({ + type: 'csv', + csv: 'title\nWrapped', + }); + }); + + it('allows an import-data directory name as the wrapper directory', async () => { + const file = await zipFile('posts.zip', (archive) => { + archive.file('content/posts.csv', 'title\nWrapped'); + }); + + await expect(inspectImportArchive(file)).resolves.toMatchObject({ + type: 'csv', + csv: 'title\nWrapped', + }); + }); + + it('leaves ZIPs without a data CSV on the legacy import path', async () => { + const json = await zipFile('export.zip', (archive) => { + archive.file('ghost-import.json', '{}'); + archive.file('content/files/attachment.csv', 'download,only'); + }); + + await expect(inspectImportArchive(json)).resolves.toEqual({ type: 'legacy' }); + }); + + it('rejects a data CSV nested below more than one wrapper directory', async () => { + const file = await zipFile('posts.zip', (archive) => { + archive.file('export/2024/posts.csv', 'title\nToo deep'); + }); + + await expect(inspectImportArchive(file)).rejects.toThrow( + 'CSV files must be at the ZIP root or inside one wrapper directory', + ); + }); + + it('rejects multiple data CSV files', async () => { + const file = await zipFile('posts.zip', (archive) => { + archive.file('one.csv', 'title\nOne'); + archive.file('two.csv', 'title\nTwo'); + }); + + await expect(inspectImportArchive(file)).rejects.toThrow('only one CSV file'); + }); + + it('rejects CSV mixed with another import data format', async () => { + const file = await zipFile('posts.zip', (archive) => { + archive.file('posts.csv', 'title\nOne'); + archive.file('posts.json', '{}'); + }); + + await expect(inspectImportArchive(file)).rejects.toThrow( + 'cannot contain CSV, JSON, or Markdown import files together', + ); + }); + + it.each(['export/2024/ghost.json', 'export/2024/posts.md'])( + 'rejects CSV mixed with deeply nested import data at %s', + async (competingName) => { + const file = await zipFile('posts.zip', (archive) => { + archive.file('posts.csv', 'title\nOne'); + archive.file(competingName, 'competing import'); + }); + + await expect(inspectImportArchive(file)).rejects.toThrow( + 'cannot contain CSV, JSON, or Markdown import files together', + ); + }, + ); + + it('keeps a JSON-only ZIP on the legacy import path regardless of nesting', async () => { + const file = await zipFile('export.zip', (archive) => { + archive.file('export/2024/ghost.json', '{}'); + }); + + await expect(inspectImportArchive(file)).resolves.toEqual({ type: 'legacy' }); + }); + + it('rejects content split between the root and a wrapper directory', async () => { + const file = await zipFile('posts.zip', (archive) => { + archive.file('export/posts.csv', 'title\nOne'); + archive.file('content/files/attachment.csv', 'download,only'); + }); + + await expect(inspectImportArchive(file)).rejects.toThrow('Invalid ZIP file structure'); + }); + + it('ignores macOS metadata entries', async () => { + const file = await zipFile('posts.zip', (archive) => { + archive.file('posts.csv', 'title\nOne'); + archive.file('__MACOSX/._posts.csv', 'metadata'); + }); + + await expect(inspectImportArchive(file)).resolves.toMatchObject({ type: 'csv' }); + }); + + it('rejects corrupt ZIP data with a stable error', async () => { + const file = new File(['not a zip'], 'posts.zip', { type: 'application/zip' }); + + await expect(inspectImportArchive(file)).rejects.toThrow('Unable to read ZIP file'); + }); +}); diff --git a/apps/admin/src/settings/advanced/migration-tools/content-import/archive.ts b/apps/admin/src/settings/advanced/migration-tools/content-import/archive.ts new file mode 100644 index 000000000000..d02c94c93546 --- /dev/null +++ b/apps/admin/src/settings/advanced/migration-tools/content-import/archive.ts @@ -0,0 +1,133 @@ +import JSZip from 'jszip'; + +const DATA_EXTENSIONS = new Set(['csv', 'json', 'md', 'markdown']); + +export type ImportArchiveContents = { type: 'legacy' } | { type: 'csv'; csv: string; name: string }; + +export class ImportArchiveError extends Error {} + +export async function inspectImportArchive(file: File): Promise { + let archive: JSZip; + try { + archive = await JSZip.loadAsync(file); + } catch { + throw new ImportArchiveError('Unable to read ZIP file. Please select a valid ZIP file.'); + } + + const files = Object.values(archive.files).filter((entry) => !entry.dir); + const dataFiles = files.filter(isDataFile); + const csvFiles = dataFiles.filter((entry) => extensionOf(entry.name) === 'csv'); + + if (files.some(isUnsupportedDataCSV)) { + throw new ImportArchiveError( + 'CSV files must be at the ZIP root or inside one wrapper directory.', + ); + } + + if (csvFiles.length === 0) { + return { type: 'legacy' }; + } + + if (csvFiles.length > 1) { + throw new ImportArchiveError( + 'ZIP files can contain only one CSV file. Remove the extra CSV files and try again.', + ); + } + + if (dataFiles.length > 1 || files.some(isUnsupportedCompetingDataFile)) { + throw new ImportArchiveError( + 'ZIP files cannot contain CSV, JSON, or Markdown import files together. Keep only the CSV file and try again.', + ); + } + + validateWrapper(files, csvFiles[0]); + + return { + type: 'csv', + csv: await csvFiles[0].async('text'), + name: csvFiles[0].name, + }; +} + +function validateWrapper(files: JSZip.JSZipObject[], csv: JSZip.JSZipObject): void { + const csvParts = csv.name.split('/').filter(Boolean); + if (csvParts.length !== 2) { + return; + } + + const wrapper = csvParts[0]; + const outsideWrapper = files.some((entry) => { + const parts = entry.name.split('/').filter(Boolean); + return !isMetadata(parts) && parts[0] !== wrapper; + }); + if (outsideWrapper) { + throw new ImportArchiveError('Invalid ZIP file structure.'); + } +} + +function isDataFile(entry: JSZip.JSZipObject): boolean { + if (entry.dir) { + return false; + } + + const parts = entry.name.split('/').filter(Boolean); + if (parts.length === 0 || isMetadata(parts)) { + return false; + } + + if (!DATA_EXTENSIONS.has(extensionOf(entry.name))) { + return false; + } + + if (parts.length === 1) { + return true; + } + + return parts.length === 2; +} + +function isUnsupportedDataCSV(entry: JSZip.JSZipObject): boolean { + if (entry.dir || extensionOf(entry.name) !== 'csv') { + return false; + } + + const parts = entry.name.split('/').filter(Boolean); + if (parts.length === 0 || isMetadata(parts) || isFileAttachment(parts)) { + return false; + } + + return !isDataFile(entry); +} + +function isUnsupportedCompetingDataFile(entry: JSZip.JSZipObject): boolean { + const extension = extensionOf(entry.name); + if (entry.dir || extension === 'csv' || !DATA_EXTENSIONS.has(extension)) { + return false; + } + + const parts = entry.name.split('/').filter(Boolean); + if (parts.length === 0 || isMetadata(parts) || isFileAttachment(parts)) { + return false; + } + + return !isDataFile(entry); +} + +function isFileAttachment(parts: string[]): boolean { + const assetParts = ['content', 'files'].includes(parts[0]) ? [...parts] : parts.slice(1); + if (assetParts[0] === 'content') { + assetParts.shift(); + } + + return assetParts.length > 1 && assetParts[0] === 'files'; +} + +function extensionOf(fileName: string): string { + return fileName.split('.').pop()?.toLowerCase() ?? ''; +} + +function isMetadata(parts: string[]): boolean { + return ( + parts[0] === '__MACOSX' || parts.some((part) => part.startsWith('._') || part === '.DS_Store') + ); +} diff --git a/apps/admin/src/settings/advanced/migration-tools/universal-import-modal.test.tsx b/apps/admin/src/settings/advanced/migration-tools/universal-import-modal.test.tsx index 6f2c6887150c..3ee66a3c2861 100644 --- a/apps/admin/src/settings/advanced/migration-tools/universal-import-modal.test.tsx +++ b/apps/admin/src/settings/advanced/migration-tools/universal-import-modal.test.tsx @@ -2,15 +2,27 @@ import { useState } from 'react'; import UniversalImportModal from '@/settings/advanced/migration-tools/universal-import-modal'; import { ConfirmationProvider } from '@/settings/providers/confirmation-provider'; import { act, fireEvent, render, screen, waitFor } from '@testing-library/react'; - -const { mockImportContent, mockImportContentCSV, mockUseFeatureFlag, mockHandleError } = vi.hoisted( - () => ({ - mockImportContent: vi.fn(), - mockImportContentCSV: vi.fn(), - mockUseFeatureFlag: vi.fn(), - mockHandleError: vi.fn(), - }), -); +import JSZip from 'jszip'; + +const { + mockImportContent, + mockImportContentCSV, + mockUseFeatureFlag, + mockHandleError, + mockToastError, +} = vi.hoisted(() => ({ + mockImportContent: vi.fn(), + mockImportContentCSV: vi.fn(), + mockUseFeatureFlag: vi.fn(), + mockHandleError: vi.fn(), + mockToastError: vi.fn(), +})); + +vi.mock('sonner', () => ({ + toast: { + error: mockToastError, + }, +})); vi.mock('@tryghost/admin-x-framework/api/db', async () => { const actual = await vi.importActual( @@ -77,6 +89,13 @@ describe('UniversalImportModal', () => { }); }; + const zipFile = async (build: (archive: JSZip) => void) => { + const archive = new JSZip(); + build(archive); + const bytes = await archive.generateAsync({ type: 'arraybuffer' }); + return new File([bytes], 'import.zip', { type: 'application/zip' }); + }; + it('reads the csvContentImporter flag', async () => { mockUseFeatureFlag.mockReturnValue(false); showModal(); @@ -204,6 +223,61 @@ describe('UniversalImportModal', () => { expect(mockImportContentCSV).not.toHaveBeenCalled(); }); + it('still sends JSON ZIP files to the db import when csvContentImporter is enabled', async () => { + mockUseFeatureFlag.mockReturnValue(true); + showModal(); + + const file = await zipFile((archive) => archive.file('ghost-import.json', '{}')); + await dropFile(file); + + await waitFor(() => expect(mockImportContent).toHaveBeenCalledWith(file)); + expect(mockImportContentCSV).not.toHaveBeenCalled(); + }); + + it('maps the single CSV in a ZIP and uploads the original archive', async () => { + mockUseFeatureFlag.mockReturnValue(true); + showModal(); + + const file = await zipFile((archive) => { + archive.file('export/posts.csv', 'title,html\nFrom ZIP,

Body

'); + archive.file('export/content/files/attachment.csv', 'download,only'); + }); + await dropFile(file); + + expect(await screen.findByText('Map CSV fields')).toBeInTheDocument(); + expect(screen.getByText('From ZIP')).toBeInTheDocument(); + fireEvent.click(screen.getByRole('button', { name: 'Import' })); + + await waitFor(() => + expect(mockImportContentCSV).toHaveBeenCalledWith({ + file, + mapping: { title: 'title', html: 'html' }, + }), + ); + expect(mockImportContent).not.toHaveBeenCalled(); + }); + + it('rejects multiple data CSVs before uploading the ZIP', async () => { + mockUseFeatureFlag.mockReturnValue(true); + showModal(); + + const file = await zipFile((archive) => { + archive.file('one.csv', 'title\nOne'); + archive.file('two.csv', 'title\nTwo'); + }); + await dropFile(file); + + await waitFor(() => + expect(mockToastError).toHaveBeenCalledWith( + 'ZIP files can contain only one CSV file. Remove the extra CSV files and try again.', + ), + ); + expect(mockHandleError).not.toHaveBeenCalled(); + expect(mockImportContent).not.toHaveBeenCalled(); + expect(mockImportContentCSV).not.toHaveBeenCalled(); + expect(screen.getByTestId('universal-import-modal')).toBeInTheDocument(); + }); + it('surfaces an error and keeps the modal open when the import fails', async () => { const error = new Error('Import failed'); mockUseFeatureFlag.mockReturnValue(true); diff --git a/apps/admin/src/settings/advanced/migration-tools/universal-import-modal.tsx b/apps/admin/src/settings/advanced/migration-tools/universal-import-modal.tsx index 1453d5d4f1d7..68ff015bc8c7 100644 --- a/apps/admin/src/settings/advanced/migration-tools/universal-import-modal.tsx +++ b/apps/admin/src/settings/advanced/migration-tools/universal-import-modal.tsx @@ -3,13 +3,15 @@ import { Button, Dropzone } from '@tryghost/shade/components'; import { ExternalLink } from 'lucide-react'; import { Inline, Stack } from '@tryghost/shade/primitives'; import { SettingsModal } from '@tryghost/shade/patterns'; +import { toast } from 'sonner'; import { useConfirmation } from '@/settings/providers/confirmation-context'; import { useFeatureFlag, useHandleError } from '@tryghost/admin-x-framework/hooks'; import { useImportContent } from '@tryghost/admin-x-framework/api/db'; import { useImportContentCSV } from '@tryghost/admin-x-framework/api/posts'; import { ContentFieldMapping } from './content-import/mapping'; import { MappingStep } from './content-import/mapping-step'; -import { columnsOf, readCSV } from './content-import/csv'; +import { columnsOf, parseCSV, readCSV } from './content-import/csv'; +import { ImportArchiveError, inspectImportArchive } from './content-import/archive'; interface CSVImportState { file: File; @@ -49,10 +51,28 @@ const UniversalImportModal: React.FC<{ onClose: () => void }> = ({ onClose }) => const importFile = async (file: File) => { const isCSV = csvContentImporter && file.name.toLowerCase().endsWith('.csv'); - if (isCSV) { + const isZIP = csvContentImporter && file.name.toLowerCase().endsWith('.zip'); + if (isCSV || isZIP) { setReading(true); try { - const rows = await readCSV(file); + let rows: Record[]; + if (isZIP) { + const contents = await inspectImportArchive(file); + if (contents.type === 'legacy') { + setReading(false); + setUploading(true); + try { + await importContent(file); + finishImport(false); + } finally { + setUploading(false); + } + return; + } + rows = parseCSV(contents.csv); + } else { + rows = await readCSV(file); + } if (rows.length === 0) { throw new Error('File is empty, nothing to import. Please select a different file.'); } @@ -63,7 +83,11 @@ const UniversalImportModal: React.FC<{ onClose: () => void }> = ({ onClose }) => sampleIndex: 0, }); } catch (error) { - handleError(error); + if (error instanceof ImportArchiveError) { + toast.error(error.message); + } else { + handleError(error); + } } finally { setReading(false); } @@ -174,7 +198,7 @@ const UniversalImportModal: React.FC<{ onClose: () => void }> = ({ onClose }) => {uploading ? ( 'Uploading...' ) : reading ? ( - 'Reading CSV...' + 'Reading import file...' ) : ( <> Select any {csvContentImporter ? 'JSON, zip or CSV' : 'JSON or zip'} file that diff --git a/ghost/core/core/server/adapters/storage/LocalStorageBase.ts b/ghost/core/core/server/adapters/storage/LocalStorageBase.ts index 25268b90a838..3a576269bc2e 100644 --- a/ghost/core/core/server/adapters/storage/LocalStorageBase.ts +++ b/ghost/core/core/server/adapters/storage/LocalStorageBase.ts @@ -95,9 +95,10 @@ class LocalStorageBase extends StorageBase { * * @param targetDir absolute or relative directory * @param fileName file name to normalize and append - * @returns resolved absolute path inside storagePath + * @param allowRoot whether storagePath itself is a valid result + * @returns resolved absolute path inside storagePath, or storagePath when explicitly allowed */ - _resolveAndValidateStoragePath(targetDir?: string, fileName?: string): string { + _resolveAndValidateStoragePath(targetDir?: string, fileName?: string, allowRoot = false): string { const resolvedRoot = path.resolve(this.storagePath); // Resolve targetDir: if already inside storage root use as-is, otherwise treat as relative @@ -123,10 +124,12 @@ class LocalStorageBase extends StorageBase { resolvedPath = resolvedBase; } - // Validate the resolved path is strictly inside the storage root (not equal to it) + // Validate the resolved path is inside the storage root. Callers which operate on files + // keep the stricter default; save() may target the root because the generated filename is + // validated separately before anything is written. const relative = path.relative(resolvedRoot, resolvedPath); if ( - relative === '' || + (!allowRoot && relative === '') || relative === '..' || relative.startsWith('..' + path.sep) || path.isAbsolute(relative) @@ -145,7 +148,7 @@ class LocalStorageBase extends StorageBase { */ async save(file: StorageFile, targetDir?: string): Promise { targetDir = targetDir - ? this._resolveAndValidateStoragePath(targetDir) + ? this._resolveAndValidateStoragePath(targetDir, undefined, true) : this.getTargetDir(this.storagePath); const targetFilename = await this.getUniqueFileName(file, targetDir); diff --git a/ghost/core/core/server/api/endpoints/db.js b/ghost/core/core/server/api/endpoints/db.js index 90b616892fad..376155a555ec 100644 --- a/ghost/core/core/server/api/endpoints/db.js +++ b/ghost/core/core/server/api/endpoints/db.js @@ -5,7 +5,7 @@ const exporter = require('../../data/exporter'); const importer = require('../../data/importer'); const mediaInliner = require('../../services/media-inliner'); const errors = require('@tryghost/errors'); -const { pool } = require('@tryghost/promise'); +const { promisePool } = require('../../lib/promise-pool'); const models = require('../../models'); const settingsCache = require('../../../shared/settings-cache'); @@ -157,7 +157,7 @@ const controller = { try { const allPosts = await models.Post.findAll(queryOpts); - await pool( + await promisePool( allPosts.map((post) => () => { return models.Post.destroy(Object.assign({ id: post.id }, queryOpts)); }), @@ -166,7 +166,7 @@ const controller = { const allTags = await models.Tag.findAll(queryOpts); - await pool( + await promisePool( allTags.map((tag) => () => { return models.Tag.destroy(Object.assign({ id: tag.id }, queryOpts)); }), diff --git a/ghost/core/core/server/api/endpoints/posts.js b/ghost/core/core/server/api/endpoints/posts.js index de51daf59999..4af5409fe3b8 100644 --- a/ghost/core/core/server/api/endpoints/posts.js +++ b/ghost/core/core/server/api/endpoints/posts.js @@ -134,6 +134,7 @@ const controller = { // the 202. const { importId, total } = await contentImportService.importCSV({ filePath: frame.file.path, + fileName: frame.file.name, mapping: frame.data.mapping, }); return { meta: { import_id: importId, total } }; diff --git a/ghost/core/core/server/data/importer/content-files.ts b/ghost/core/core/server/data/importer/content-files.ts new file mode 100644 index 000000000000..9b013cf8f880 --- /dev/null +++ b/ghost/core/core/server/data/importer/content-files.ts @@ -0,0 +1,54 @@ +import config from '../../../shared/config'; +import urlUtils from '../../../shared/url-utils'; + +// Keep adapter loading on the existing CommonJS path so the configured adapter +// and its base class share the same module instance during the legacy /db/ import. +const adapterManager = require('../../services/adapter-manager').default; +const ImageHandler = require('./handlers/image'); +const ImporterContentFileHandler = require('./handlers/importer-content-file-handler'); +const ContentFileImporter = require('./importers/content-file-importer'); + +export function createContentFileHandlers() { + const mediaStorage = adapterManager.getAdapter('storage:media'); + const fileStorage = adapterManager.getAdapter('storage:files'); + + return [ + ImageHandler, + new ImporterContentFileHandler({ + type: 'media', + // These broad directory names preserve the existing /db/ archive behavior. + directories: ['media', 'content'], + ignoreRootFolderFiles: true, + extensions: config.get('uploads').media.extensions, + contentTypes: config.get('uploads').media.contentTypes, + urlUtils, + storage: mediaStorage, + }), + new ImporterContentFileHandler({ + type: 'files', + directories: ['files', 'content'], + ignoreRootFolderFiles: true, + extensions: config.get('uploads').files.extensions, + contentTypes: config.get('uploads').files.contentTypes, + urlUtils, + storage: fileStorage, + }), + ]; +} + +export function createContentFileImporters() { + return [ + new ContentFileImporter({ + type: 'images', + store: adapterManager.getAdapter('storage:images'), + }), + new ContentFileImporter({ + type: 'media', + store: adapterManager.getAdapter('storage:media'), + }), + new ContentFileImporter({ + type: 'files', + store: adapterManager.getAdapter('storage:files'), + }), + ]; +} diff --git a/ghost/core/core/server/data/importer/handlers/image.js b/ghost/core/core/server/data/importer/handlers/image.js index f6d4859fc44f..6740546bfaa9 100644 --- a/ghost/core/core/server/data/importer/handlers/image.js +++ b/ghost/core/core/server/data/importer/handlers/image.js @@ -30,7 +30,7 @@ ImageHandler = { file.originalPath = noBaseDir; file.name = noGhostDirs; - file.targetDir = path.join(config.getContentPath('images'), path.dirname(noGhostDirs)); + file.targetDir = path.join(store.storagePath, path.dirname(noGhostDirs)); return file; }); @@ -41,7 +41,7 @@ ImageHandler = { '/', urlUtils.getSubdir(), store.staticFileURLPrefix, - path.relative(config.getContentPath('images'), targetFilename), + path.relative(store.storagePath, targetFilename), ); return image; diff --git a/ghost/core/core/server/data/importer/import-archive.ts b/ghost/core/core/server/data/importer/import-archive.ts new file mode 100644 index 000000000000..f11d7811fc45 --- /dev/null +++ b/ghost/core/core/server/data/importer/import-archive.ts @@ -0,0 +1,176 @@ +import crypto from 'node:crypto'; +import os from 'node:os'; +import path from 'node:path'; +import fs from 'fs-extra'; +import { globSync } from 'glob'; + +const { extract } = require('@tryghost/zip') as { + extract(filePath: string, directory: string): Promise; +}; +const tpl = require('@tryghost/tpl'); +const errors = require('@tryghost/errors'); + +const messages = { + noContentToImport: 'Zip did not include any content to import.', + invalidZipStructure: 'Invalid zip file structure.', + invalidZipFileBaseDirectory: 'Invalid zip file: base directory read failed', + invalidZipFileNameEncoding: 'The uploaded zip could not be read', + invalidZipFileNameEncodingContext: 'The filename was too long or contained invalid characters', + invalidZipFileNameEncodingHelp: + 'Remove any special characters from the file name, or alternatively try another archiving tool if using MacOS Archive Utility', +}; + +const ROOT_ONLY = 0; +const ROOT_OR_SINGLE_DIR = 1; +const ALL_DIRS = 2; + +export interface ImportArchiveFile { + name: string; + path: string; +} + +interface ImportArchiveOptions { + extensions: string[]; + directories: string[]; +} + +interface ImportArchiveDependencies { + extract?: (filePath: string, directory: string) => Promise; +} + +export default class ImportArchive { + readonly extensions: string[]; + readonly directories: string[]; + private readonly extractArchive: (filePath: string, directory: string) => Promise; + + constructor( + { extensions, directories }: ImportArchiveOptions, + { extract: extractArchive = extract }: ImportArchiveDependencies = {}, + ) { + this.extensions = extensions; + this.directories = directories; + this.extractArchive = extractArchive; + } + + getGlobPattern(items: string[]): string { + return `+(${items.join('|')})`; + } + + getExtensionGlob(extensions: string[], level?: number): string { + const prefix = level === ALL_DIRS ? '**/*' : level === ROOT_OR_SINGLE_DIR ? '{*/*,*}' : '*'; + + return prefix + this.getGlobPattern(extensions); + } + + getDirectoryGlob(directories: string[], level?: number): string { + const prefix = level === ALL_DIRS ? '**/' : level === ROOT_OR_SINGLE_DIR ? '{*/,}' : ''; + + return prefix + this.getGlobPattern(directories); + } + + isValid(directory: string): boolean { + const extMatchesBase = globSync(this.getExtensionGlob(this.extensions, ROOT_OR_SINGLE_DIR), { + cwd: directory, + nocase: true, + }); + + const extMatchesAll = globSync(this.getExtensionGlob(this.extensions, ALL_DIRS), { + cwd: directory, + nocase: true, + }); + + const dirMatches = this.directories.length + ? globSync(this.getDirectoryGlob(this.directories, ROOT_OR_SINGLE_DIR), { + cwd: directory, + }) + : []; + + if (extMatchesBase.length > 0 || (dirMatches.length > 0 && extMatchesAll.length > 0)) { + return true; + } + + if (extMatchesAll.length < 1) { + throw new errors.UnsupportedMediaTypeError({ message: tpl(messages.noContentToImport) }); + } + + throw new errors.UnsupportedMediaTypeError({ message: tpl(messages.invalidZipStructure) }); + } + + getBaseDirectory(directory: string): string | undefined { + const extMatches = globSync(this.getExtensionGlob(this.extensions, ROOT_ONLY), { + cwd: directory, + nocase: true, + }); + + const dirMatches = this.directories.length + ? globSync(this.getDirectoryGlob(this.directories, ROOT_ONLY), { + cwd: directory, + nocase: true, + }) + : []; + + if (extMatches.length > 0 || dirMatches.length > 0) { + return; + } + + const extMatchesAll = globSync(this.getExtensionGlob(this.extensions, ALL_DIRS), { + cwd: directory, + nocase: true, + }); + const firstMatch = extMatchesAll[0]; + if (!firstMatch) { + throw new errors.ValidationError({ message: tpl(messages.invalidZipFileBaseDirectory) }); + } + + return firstMatch.split('/')[0]; + } + + getFiles(directory: string, extensions: string[]): ImportArchiveFile[] { + const globPattern = this.getExtensionGlob(extensions, ALL_DIRS); + return globSync(globPattern, { cwd: directory, nocase: true }).map((file) => ({ + name: file, + path: path.join(directory, file), + })); + } + + async extract(filePath: string): Promise { + const tmpDir = path.join(os.tmpdir(), crypto.randomUUID()); + + try { + await this.extractArchive(filePath, tmpDir); + + const files = globSync('**/*', { cwd: tmpDir, nodir: true }); + await Promise.all(files.map((file) => fs.chmod(path.join(tmpDir, file), 0o644))); + } catch (error) { + await fs.remove(tmpDir).catch(() => {}); + const message = messageOf(error); + + if (message.startsWith('ENAMETOOLONG:')) { + throw new errors.UnsupportedMediaTypeError({ + message: tpl(messages.invalidZipFileNameEncoding), + context: tpl(messages.invalidZipFileNameEncodingContext), + help: tpl(messages.invalidZipFileNameEncodingHelp), + code: 'INVALID_ZIP_FILE_NAME_ENCODING', + }); + } + + if ( + message.includes('end of central directory record signature not found') || + message.includes('invalid comment length') + ) { + throw new errors.UnsupportedMediaTypeError({ + message: tpl(messages.invalidZipFileNameEncoding), + code: 'INVALID_ZIP_FILE', + }); + } + + throw error; + } + + return tmpDir; + } +} + +function messageOf(error: unknown): string { + return error instanceof Error ? error.message : ''; +} diff --git a/ghost/core/core/server/data/importer/import-manager.js b/ghost/core/core/server/data/importer/import-manager.js index 25c47255da60..4763144405ba 100644 --- a/ghost/core/core/server/data/importer/import-manager.js +++ b/ghost/core/core/server/data/importer/import-manager.js @@ -1,31 +1,21 @@ const _ = require('lodash'); const fs = require('fs-extra'); const path = require('path'); -const os = require('os'); -const { globSync } = require('glob'); -const crypto = require('crypto'); const config = require('../../../shared/config'); -const { extract } = require('@tryghost/zip'); const tpl = require('@tryghost/tpl'); const debug = require('@tryghost/debug')('import-manager'); const logging = require('@tryghost/logging'); const errors = require('@tryghost/errors'); -const ImageHandler = require('./handlers/image'); -const ImporterContentFileHandler = require('./handlers/importer-content-file-handler'); const RevueHandler = require('./handlers/revue'); const JSONHandler = require('./handlers/json'); const MarkdownHandler = require('./handlers/markdown'); -const ContentFileImporter = require('./importers/content-file-importer'); const RevueImporter = require('./importers/importer-revue'); const DataImporter = require('./importers/data'); const urlUtils = require('../../../shared/url-utils').default; const { GhostMailer } = require('../../services/mail'); const jobManager = require('../../services/jobs'); -const adapterManager = require('../../services/adapter-manager').default; - -const mediaStorage = adapterManager.getAdapter('storage:media'); -const imageStorage = adapterManager.getAdapter('storage:images'); -const fileStorage = adapterManager.getAdapter('storage:files'); +const ImportArchive = require('./import-archive').default; +const { createContentFileHandlers, createContentFileImporters } = require('./content-files'); const { emailTemplate } = require('./email-template'); const ghostMailer = new GhostMailer(); @@ -36,21 +26,9 @@ const messages = { context: 'Your site will continue to work as expected', }, noContentToImport: 'Zip did not include any content to import.', - invalidZipStructure: 'Invalid zip file structure.', - invalidZipFileBaseDirectory: 'Invalid zip file: base directory read failed', zipContainsMultipleDataFormats: 'Zip file contains multiple data formats. Please split up and import separately.', - invalidZipFileNameEncoding: 'The uploaded zip could not be read', - invalidZipFileNameEncodingContext: 'The filename was too long or contained invalid characters', - invalidZipFileNameEncodingHelp: - 'Remove any special characters from the file name, or alternatively try another archiving tool if using MacOS Archive Utility', }; - -// Glob levels -const ROOT_ONLY = 0; - -const ROOT_OR_SINGLE_DIR = 1; -const ALL_DIRS = 2; let defaults = { extensions: ['.zip'], contentTypes: ['application/zip', 'application/x-zip-compressed'], @@ -59,68 +37,23 @@ let defaults = { class ImportManager { constructor() { - const mediaHandler = new ImporterContentFileHandler({ - type: 'media', - // @NOTE: making the second parameter strict folder "content/media" brakes the glob pattern - // in the importer, so we need to keep it as general "content" unless - // it becomes a strict requirement - directories: ['media', 'content'], - ignoreRootFolderFiles: true, - extensions: config.get('uploads').media.extensions, - contentTypes: config.get('uploads').media.contentTypes, - urlUtils: urlUtils, - storage: mediaStorage, - }); - - const filesHandler = new ImporterContentFileHandler({ - type: 'files', - // @NOTE: making the second parameter strict folder "content/files" brakes the glob pattern - // in the importer, so we need to keep it as general "content" unless - // it becomes a strict requirement - directories: ['files', 'content'], - ignoreRootFolderFiles: true, - extensions: config.get('uploads').files.extensions, - contentTypes: config.get('uploads').files.contentTypes, - urlUtils: urlUtils, - storage: fileStorage, - }); - - const imageImporter = new ContentFileImporter({ - type: 'images', - store: imageStorage, - }); - const mediaImporter = new ContentFileImporter({ - type: 'media', - store: mediaStorage, - }); - - const contentFilesImporter = new ContentFileImporter({ - type: 'files', - store: fileStorage, - }); + const contentFileHandlers = createContentFileHandlers(); + const contentFileImporters = createContentFileImporters(); /** * @type {Importer[]} importers */ - this.importers = [ - imageImporter, - mediaImporter, - contentFilesImporter, - RevueImporter, - DataImporter, - ]; + this.importers = [...contentFileImporters, RevueImporter, DataImporter]; /** * @type {Handler[]} */ - this.handlers = [ - ImageHandler, - mediaHandler, - filesHandler, - RevueHandler, - JSONHandler, - MarkdownHandler, - ]; + this.handlers = [...contentFileHandlers, RevueHandler, JSONHandler, MarkdownHandler]; + + this.archive = new ImportArchive({ + extensions: this.getExtensions(), + directories: this.getDirectories(), + }); // Keep track of file to cleanup at the end /** @@ -159,17 +92,7 @@ class ImportManager { * @returns {string} */ getGlobPattern(items) { - return ( - '+(' + - _.reduce( - items, - function (memo, ext) { - return memo !== '' ? memo + '|' + ext : ext; - }, - '', - ) + - ')' - ); + return this.archive.getGlobPattern(items); } /** @@ -178,9 +101,7 @@ class ImportManager { * @returns {string} */ getExtensionGlob(extensions, level) { - const prefix = level === ALL_DIRS ? '**/*' : level === ROOT_OR_SINGLE_DIR ? '{*/*,*}' : '*'; - - return prefix + this.getGlobPattern(extensions); + return this.archive.getExtensionGlob(extensions, level); } /** @@ -190,9 +111,7 @@ class ImportManager { * @returns {string} */ getDirectoryGlob(directories, level) { - const prefix = level === ALL_DIRS ? '**/' : level === ROOT_OR_SINGLE_DIR ? '{*/,}' : ''; - - return prefix + this.getGlobPattern(directories); + return this.archive.getDirectoryGlob(directories, level); } /** @@ -212,31 +131,7 @@ class ImportManager { * @returns {boolean} */ isValidZip(directory) { - // Globs match content in the root or inside a single directory - const extMatchesBase = globSync( - this.getExtensionGlob(this.getExtensions(), ROOT_OR_SINGLE_DIR), - { cwd: directory, nocase: true }, - ); - - const extMatchesAll = globSync(this.getExtensionGlob(this.getExtensions(), ALL_DIRS), { - cwd: directory, - nocase: true, - }); - - const dirMatches = globSync(this.getDirectoryGlob(this.getDirectories(), ROOT_OR_SINGLE_DIR), { - cwd: directory, - }); - - // If this folder contains importable files or a content or images directory - if (extMatchesBase.length > 0 || (dirMatches.length > 0 && extMatchesAll.length > 0)) { - return true; - } - - if (extMatchesAll.length < 1) { - throw new errors.UnsupportedMediaTypeError({ message: tpl(messages.noContentToImport) }); - } - - throw new errors.UnsupportedMediaTypeError({ message: tpl(messages.invalidZipStructure) }); + return this.archive.isValid(directory); } /** @@ -245,37 +140,8 @@ class ImportManager { * @returns {Promise} full path to the extracted folder */ async extractZip(filePath) { - const tmpDir = path.join(os.tmpdir(), crypto.randomUUID()); + const tmpDir = await this.archive.extract(filePath); this.fileToDelete = tmpDir; - - try { - await extract(filePath, tmpDir); - - // Set permissions for all extracted files - const files = globSync('**/*', { cwd: tmpDir, nodir: true }); - await Promise.all(files.map((file) => fs.chmod(path.join(tmpDir, file), 0o644))); - } catch (err) { - if (err.message.startsWith('ENAMETOOLONG:')) { - // The file was probably zipped with MacOS zip utility. Which doesn't correctly set UTF-8 encoding flag. - // This causes ENAMETOOLONG error on Linux, because the resulting filename length is too long when decoded using the default string encoder. - throw new errors.UnsupportedMediaTypeError({ - message: tpl(messages.invalidZipFileNameEncoding), - context: tpl(messages.invalidZipFileNameEncodingContext), - help: tpl(messages.invalidZipFileNameEncodingHelp), - code: 'INVALID_ZIP_FILE_NAME_ENCODING', - }); - } else if ( - err.message.includes('end of central directory record signature not found') || - err.message.includes('invalid comment length') - ) { - // This comes from Yauzl when the zip is invalid - throw new errors.UnsupportedMediaTypeError({ - message: tpl(messages.invalidZipFileNameEncoding), - code: 'INVALID_ZIP_FILE', - }); - } - throw err; - } return tmpDir; } @@ -287,10 +153,7 @@ class ImportManager { * @returns {File[]} Files */ getFilesFromZip(handler, directory) { - const globPattern = this.getExtensionGlob(handler.extensions, ALL_DIRS); - return _.map(globSync(globPattern, { cwd: directory, nocase: true }), function (file) { - return { name: file, path: path.join(directory, file) }; - }); + return this.archive.getFiles(directory, handler.extensions); } /** @@ -299,32 +162,7 @@ class ImportManager { * @returns {string} */ getBaseDirectory(directory) { - // Globs match root level only - const extMatches = globSync(this.getExtensionGlob(this.getExtensions(), ROOT_ONLY), { - cwd: directory, - nocase: true, - }); - - const dirMatches = globSync(this.getDirectoryGlob(this.getDirectories(), ROOT_ONLY), { - cwd: directory, - nocase: true, - }); - let extMatchesAll; - - // There is no base directory - if (extMatches.length > 0 || dirMatches.length > 0) { - return; - } - // There is a base directory, grab it from any ext match - extMatchesAll = globSync(this.getExtensionGlob(this.getExtensions(), ALL_DIRS), { - cwd: directory, - nocase: true, - }); - if (extMatchesAll.length < 1 || extMatchesAll[0].split('/').length < 1) { - throw new errors.ValidationError({ message: tpl(messages.invalidZipFileBaseDirectory) }); - } - - return extMatchesAll[0].split('/')[0]; + return this.archive.getBaseDirectory(directory); } /** diff --git a/ghost/core/core/server/data/importer/importers/content-file-importer.js b/ghost/core/core/server/data/importer/importers/content-file-importer.js index 2d6a0345982d..0538e2b8cff6 100644 --- a/ghost/core/core/server/data/importer/importers/content-file-importer.js +++ b/ghost/core/core/server/data/importer/importers/content-file-importer.js @@ -1,4 +1,5 @@ const _ = require('lodash'); +const path = require('node:path'); let replaceImage; let preProcessPosts; let preProcessTags; @@ -9,8 +10,19 @@ replaceImage = function (markdown, image) { return; } - // Normalizes to include a trailing slash if there was one - const regex = new RegExp('(/)?' + image.originalPath, 'gm'); + const originalPaths = [image.originalPath]; + if (image.originalPath.startsWith('content/')) { + originalPaths.push(image.originalPath.slice('content/'.length)); + } else { + originalPaths.unshift(`content/${image.originalPath}`); + } + + // Match canonical /content/{type} paths before their shorter archive path so + // replacing images/image.jpg cannot leave an extra /content prefix behind. + // Consume __GHOST_URL__ too: newPath already contains the configured subdir + // and the model will normalize it back to transform-ready form on write. + const escapedPaths = originalPaths.map((originalPath) => _.escapeRegExp(originalPath)); + const regex = new RegExp(`(?:__GHOST_URL__)?/?(?:${escapedPaths.join('|')})`, 'gm'); return markdown.replace(regex, image.newPath); }; @@ -31,6 +43,12 @@ preProcessPosts = function (data, contentFile) { if (post.feature_image) { post.feature_image = replaceImage(post.feature_image, contentFile); } + if (post.og_image) { + post.og_image = replaceImage(post.og_image, contentFile); + } + if (post.twitter_image) { + post.twitter_image = replaceImage(post.twitter_image, contentFile); + } }); }; @@ -114,10 +132,10 @@ class ContentFileImporter { * @param {Object[]} contentFilesData * @returns */ - doImport(contentFilesData) { + async doImport(contentFilesData) { const store = this.#store; - return Promise.all( + const results = await Promise.allSettled( contentFilesData.map(function (contentFile) { return store.save(contentFile, contentFile.targetDir).then(function (result) { return { @@ -128,6 +146,50 @@ class ContentFileImporter { }); }), ); + + const failure = results.find((result) => result.status === 'rejected'); + if (failure) { + const storedFiles = results.flatMap((result) => + result.status === 'fulfilled' ? [result.value] : [], + ); + try { + await this.rollback(storedFiles); + } catch (rollbackFailure) { + // AggregateError is the native error for preserving both storage failures; + // it is not a GhostError and therefore does not accept an options object. + // eslint-disable-next-line ghost/ghost-custom/ghost-error-usage + throw new AggregateError( + [failure.reason, rollbackFailure], + 'Asset storage failed and rollback was incomplete.', + { cause: failure.reason }, + ); + } + throw failure.reason; + } + + return results.map((result) => result.value); + } + + /** + * @param {{stored: string}[]} storedFiles + * @returns {Promise} + */ + async rollback(storedFiles) { + const store = this.#store; + const results = await Promise.allSettled( + storedFiles.map(function (storedFile) { + const storedPath = store.urlToPath(storedFile.stored); + const targetDir = path.posix.dirname(storedPath); + return store.delete( + path.posix.basename(storedPath), + targetDir === '.' ? undefined : targetDir, + ); + }), + ); + const failure = results.find((result) => result.status === 'rejected'); + if (failure) { + throw failure.reason; + } } } diff --git a/ghost/core/core/server/lib/promise-pool.ts b/ghost/core/core/server/lib/promise-pool.ts new file mode 100644 index 000000000000..47c516822a91 --- /dev/null +++ b/ghost/core/core/server/lib/promise-pool.ts @@ -0,0 +1,33 @@ +import errors from '@tryghost/errors'; +import type { Promisable } from 'type-fest'; + +const validateMaxConcurrency = (maxConcurrency: number) => { + if (maxConcurrency < 1 || !Number.isSafeInteger(maxConcurrency)) { + throw new errors.IncorrectUsageError({ + message: 'Concurrency must be a positive integer', + }); + } +}; + +/** + * Run promise-returning tasks with a bounded level of concurrency. + */ +export const promisePool = async ( + tasks: Array<() => Promisable>, + maxConcurrency: number, +): Promise => { + validateMaxConcurrency(maxConcurrency); + + const taskIterator = tasks.values(); + + const concurrency = Math.min(maxConcurrency, tasks.length); + const workers = Array(concurrency) + .fill(taskIterator) + .map(async (workerIterator) => { + for (const task of workerIterator) { + await task(); + } + }); + + await Promise.all(workers); +}; diff --git a/ghost/core/core/server/services/automations/poll.ts b/ghost/core/core/server/services/automations/poll.ts index 2317e378ff61..9c1cf5484e90 100644 --- a/ghost/core/core/server/services/automations/poll.ts +++ b/ghost/core/core/server/services/automations/poll.ts @@ -223,6 +223,7 @@ const processStep = async ({ subject: step.email_subject, }, member: { + // Note that we don't use `automation_run.member_email` here. That's only for the historical record, not for functionality. email: member.get('email'), name: member.get('name'), uuid: member.get('uuid'), diff --git a/ghost/core/core/server/services/content-import/import/assets.ts b/ghost/core/core/server/services/content-import/import/assets.ts new file mode 100644 index 000000000000..8888f58b0aca --- /dev/null +++ b/ghost/core/core/server/services/content-import/import/assets.ts @@ -0,0 +1,160 @@ +import type { PostImportRow } from './row'; +import { + createContentFileHandlers, + createContentFileImporters, +} from '../../../data/importer/content-files'; + +type AssetType = 'images' | 'media' | 'files'; + +export interface PreparedAsset { + name: string; + path: string; + originalPath: string; + newPath: string; + targetDir: string; +} + +interface StoredAsset { + originalPath: string; + newPath: string; + stored: string; +} + +interface AssetHandler { + type: AssetType; + extensions: string[]; + loadFile( + files: Array<{ name: string; path: string }>, + baseDirectory?: string, + ): Promise; +} + +interface AssetImporter { + type: AssetType; + doImport(files: PreparedAsset[]): Promise; + rollback(files: StoredAsset[]): Promise; + preProcess(importData: Record): Record; +} + +interface ArchiveFiles { + getFiles(directory: string, extensions: string[]): Array<{ name: string; path: string }>; +} + +interface PreparedAssetGroup { + type: AssetType; + files: PreparedAsset[]; + importer: AssetImporter; +} + +export interface AssetPreparationDeps { + handlers?: AssetHandler[]; + importers?: AssetImporter[]; +} + +export interface ImportAssetBatch { + readonly files: PreparedAsset[]; + store(): Promise; + rewriteRows(rows: PostImportRow[]): void; +} + +export class PreparedAssetBatch implements ImportAssetBatch { + private groups: PreparedAssetGroup[]; + + constructor(groups: PreparedAssetGroup[]) { + this.groups = groups; + } + + get files(): PreparedAsset[] { + return this.groups.flatMap((group) => group.files); + } + + async store(): Promise { + const results = await Promise.allSettled( + this.groups.map((group) => group.importer.doImport(group.files)), + ); + const failure = results.find((result) => result.status === 'rejected'); + if (failure) { + const rollbackResults = await Promise.allSettled( + results.flatMap((result, index) => + result.status === 'fulfilled' ? [this.groups[index].importer.rollback(result.value)] : [], + ), + ); + const rollbackFailures = rollbackResults.flatMap((result) => + result.status === 'rejected' ? [result.reason] : [], + ); + if (rollbackFailures.length > 0) { + // AggregateError is the native error for preserving storage and rollback + // failures; unlike GhostError it takes an iterable before its options. + // eslint-disable-next-line ghost/ghost-custom/ghost-error-usage + throw new AggregateError( + [failure.reason, ...rollbackFailures], + 'Asset storage failed and rollback was incomplete.', + { cause: failure.reason }, + ); + } + throw failure.reason; + } + } + + rewriteRows(rows: PostImportRow[]): void { + const importData: Record = { + data: { data: { posts: rows, tags: [], users: [] } }, + }; + + for (const group of this.groups) { + importData[group.type] = group.files; + group.importer.preProcess(importData); + } + } +} + +export async function prepareAssetBatch( + archive: ArchiveFiles, + directory: string, + baseDirectory?: string, + deps: AssetPreparationDeps = {}, +): Promise { + const handlers = deps.handlers ?? (createContentFileHandlers() as AssetHandler[]); + const importers = deps.importers ?? (createContentFileImporters() as AssetImporter[]); + const importerByType = new Map(importers.map((importer) => [importer.type, importer])); + + const groups = await Promise.all( + handlers.map(async (handler): Promise => { + const files = archive + .getFiles(directory, handler.extensions) + .filter((file) => belongsToAssetDirectory(file.name, handler.type, baseDirectory)); + if (files.length === 0) { + return; + } + + const importer = importerByType.get(handler.type); + if (!importer) { + return; + } + + return { + type: handler.type, + files: await handler.loadFile(files, baseDirectory), + importer, + }; + }), + ); + const presentGroups = groups.filter((group): group is PreparedAssetGroup => group !== undefined); + return presentGroups.length > 0 ? new PreparedAssetBatch(presentGroups) : undefined; +} + +export function belongsToAssetDirectory( + fileName: string, + type: AssetType, + baseDirectory?: string, +): boolean { + const parts = fileName.split('/').filter(Boolean); + if (baseDirectory && parts[0] === baseDirectory) { + parts.shift(); + } + if (parts[0] === 'content') { + parts.shift(); + } + + return parts.length > 1 && parts[0] === type; +} diff --git a/ghost/core/core/server/services/content-import/import/importer.ts b/ghost/core/core/server/services/content-import/import/importer.ts index 8dca5d2d0cfe..51827daaa9fc 100644 --- a/ghost/core/core/server/services/content-import/import/importer.ts +++ b/ghost/core/core/server/services/content-import/import/importer.ts @@ -7,7 +7,11 @@ import buildPostData, { type PostData, } from './post-data'; import type { PostImportRow } from './row'; +import type { ImportRequest } from './schema'; import type { Clock, ImportRunStore, RowOutcome } from './store'; +import type { PreparedImportSource } from './source'; + +export type { ImportRequest } from './schema'; const errors = require('@tryghost/errors'); const logging = require('@tryghost/logging'); @@ -17,11 +21,6 @@ const tpl = require('@tryghost/tpl'); // response is sent); the parsed rows are handed to an in-process background job // that writes one post per row. -export interface ImportRequest { - filePath: string; - mapping?: Record; -} - // The id is what a completion report will be looked up by. export interface ImportAccepted { importId: string; @@ -41,6 +40,7 @@ export interface PostsRepository { export type FailureReporter = (error: unknown) => void; type ReadRows = (path: string, mapping?: Record) => Promise; +type PrepareSource = (request: ImportRequest) => Promise; const messages = { unreadableFile: 'The file could not be parsed as a CSV file.', @@ -62,6 +62,7 @@ const MAX_POSTS = 100; interface ImporterDeps { readRows: ReadRows; + prepareSource?: PrepareSource; posts: PostsRepository; // A getter so the heavy html->lexical require resolves once per run getHtmlToLexical: () => HtmlToLexical; @@ -85,6 +86,7 @@ function buildImportTagNames(runId: string, timezone: string, now: Date): string class ContentCSVImporter { private _readRows: ReadRows; + private _prepareSource: PrepareSource; private _posts: PostsRepository; private _getHtmlToLexical: () => HtmlToLexical; private _getMarkdownToHtml: () => MarkdownToHtml; @@ -99,6 +101,7 @@ class ContentCSVImporter { constructor({ readRows, + prepareSource = async ({ filePath }) => ({ filePath, cleanup: async () => {} }), posts, getHtmlToLexical, getMarkdownToHtml, @@ -112,6 +115,7 @@ class ContentCSVImporter { now = () => new Date(), }: ImporterDeps) { this._readRows = readRows; + this._prepareSource = prepareSource; this._posts = posts; this._getHtmlToLexical = getHtmlToLexical; this._getMarkdownToHtml = getMarkdownToHtml; @@ -126,10 +130,12 @@ class ContentCSVImporter { } async importCSV(request: ImportRequest): Promise { + const source = await this._prepareSource(request); let rows: PostImportRow[]; try { - rows = await this._readRows(request.filePath, request.mapping); + rows = await this._readRows(source.filePath, request.mapping); } catch (error) { + await this.cleanupSource(source.cleanup); throw new errors.ValidationError({ message: tpl(messages.unreadableFile), err: error, @@ -139,6 +145,7 @@ class ContentCSVImporter { // Temporary while import state is held in memory: the durable job // system milestone removes the cap. if (rows.length > MAX_POSTS) { + await this.cleanupSource(source.cleanup); throw new errors.ValidationError({ message: tpl(messages.tooManyPosts, { max: MAX_POSTS }), }); @@ -149,11 +156,17 @@ class ContentCSVImporter { this._store.create(runId, rows.length); logLifecycle('queued'); - this._addJob({ - job: () => this.runImportJob(runId, importTagNames, rows), - offloaded: false, - name: 'content-import', - }); + try { + this._addJob({ + job: () => this.runImportJob(runId, importTagNames, rows, source), + offloaded: false, + name: 'content-import', + }); + } catch (error) { + this._store.fail(runId, messageOf(error)); + await this.cleanupSource(source.cleanup); + throw error; + } return { importId: runId, total: rows.length }; } @@ -164,6 +177,7 @@ class ContentCSVImporter { runId: string, importTagNames: string[], rows: PostImportRow[], + source: PreparedImportSource, ): Promise { const startedAt = Date.now(); logLifecycle('started'); @@ -172,6 +186,11 @@ class ContentCSVImporter { let failed = false; try { + if (source.assets) { + await source.assets.store(); + source.assets.rewriteRows(rows); + } + const htmlToLexical = this._getHtmlToLexical(); const markdownToHtml = this._getMarkdownToHtml(); const cleanHTML = this._getCleanHTML(); @@ -267,6 +286,7 @@ class ContentCSVImporter { this._report(error); this._store.fail(runId, messageOf(error)); } finally { + await this.cleanupSource(source.cleanup); const outcome = failed ? 'failed after' : 'completed in'; logLifecycle(`${outcome} ${Date.now() - startedAt}ms`); } @@ -285,6 +305,14 @@ class ContentCSVImporter { ); } } + + private async cleanupSource(cleanup: () => Promise): Promise { + try { + await cleanup(); + } catch (error) { + this._report(error); + } + } } function messageOf(error: unknown): string { diff --git a/ghost/core/core/server/services/content-import/import/schema.ts b/ghost/core/core/server/services/content-import/import/schema.ts new file mode 100644 index 000000000000..bafd1601dffd --- /dev/null +++ b/ghost/core/core/server/services/content-import/import/schema.ts @@ -0,0 +1,31 @@ +import { z } from 'zod'; +import { EDITORIAL_POST_FIELDS } from './row'; + +const importableFields = new Set(EDITORIAL_POST_FIELDS); + +export const mappingSchema = z.record(z.string(), z.string()).superRefine((mapping, ctx) => { + const targets = new Set(); + for (const [header, target] of Object.entries(mapping)) { + if (!header || header in Object.prototype) { + ctx.addIssue({ code: 'custom', message: `Invalid CSV header mapping: "${header}"` }); + } + if (target && !importableFields.has(target)) { + ctx.addIssue({ code: 'custom', message: `Unknown post field mapping: "${target}"` }); + } + if (target && targets.has(target)) { + ctx.addIssue({ code: 'custom', message: `Post field is mapped more than once: "${target}"` }); + } + targets.add(target); + } + if (!targets.has('title')) { + ctx.addIssue({ code: 'custom', message: 'Post field mapping must include "title"' }); + } +}); + +export const importRequestSchema = z.object({ + filePath: z.string().min(1), + fileName: z.string().min(1), + mapping: mappingSchema.optional(), +}); + +export type ImportRequest = z.infer; diff --git a/ghost/core/core/server/services/content-import/import/source.ts b/ghost/core/core/server/services/content-import/import/source.ts new file mode 100644 index 000000000000..72e9f60568a4 --- /dev/null +++ b/ghost/core/core/server/services/content-import/import/source.ts @@ -0,0 +1,159 @@ +import fs from 'fs-extra'; +import path from 'path'; +import { globSync } from 'glob'; +import ImportArchive from '../../../data/importer/import-archive'; +import { prepareAssetBatch, type ImportAssetBatch } from './assets'; + +const errors = require('@tryghost/errors'); + +const DATA_EXTENSIONS = new Set(['.csv', '.json', '.md', '.markdown']); + +export interface ImportSourceRequest { + filePath: string; + fileName: string; +} + +export interface PreparedImportSource { + filePath: string; + assets?: ImportAssetBatch; + cleanup(): Promise; +} + +const noCleanup = async (): Promise => {}; + +export async function prepareImportSource( + request: ImportSourceRequest, +): Promise { + const extension = path.extname(request.fileName).toLowerCase(); + if (extension === '.csv') { + return { filePath: request.filePath, cleanup: noCleanup }; + } + + if (extension !== '.zip') { + throw new errors.ValidationError({ message: 'Please select a valid CSV or ZIP file.' }); + } + + const archive = new ImportArchive({ + extensions: [...DATA_EXTENSIONS], + directories: [], + }); + const directory = await archive.extract(request.filePath); + let cleaned = false; + const cleanup = async () => { + if (!cleaned) { + cleaned = true; + await fs.remove(directory); + } + }; + + try { + archive.isValid(directory); + const baseDirectory = archive.getBaseDirectory(directory); + const entries = globSync('**/*', { cwd: directory, nodir: true, dot: true }); + const dataFiles = entries.filter(isDataFile); + const csvFiles = dataFiles.filter((file) => extensionOf(file) === '.csv'); + + if (csvFiles.length === 0) { + throw new errors.ValidationError({ + message: 'ZIP files uploaded here must contain one CSV file.', + }); + } + + if (csvFiles.length > 1) { + throw new errors.ValidationError({ + message: + 'ZIP files can contain only one CSV file. Remove the extra CSV files and try again.', + }); + } + + if (dataFiles.length > 1 || entries.some(isUnsupportedCompetingDataFile)) { + throw new errors.ValidationError({ + message: + 'ZIP files cannot contain CSV, JSON, or Markdown import files together. Keep only the CSV file and try again.', + }); + } + + validateWrapper(entries, csvFiles[0]); + const csvParts = csvFiles[0].split('/').filter(Boolean); + const csvBaseDirectory = csvParts.length === 2 ? csvParts[0] : undefined; + if (csvBaseDirectory !== baseDirectory) { + throw new errors.ValidationError({ message: 'Invalid ZIP file structure.' }); + } + + return { + filePath: path.join(directory, csvFiles[0]), + assets: await prepareAssetBatch(archive, directory, baseDirectory), + cleanup, + }; + } catch (error) { + await cleanup(); + throw error; + } +} + +function validateWrapper(entries: string[], csvFile: string): void { + const csvParts = csvFile.split('/').filter(Boolean); + if (csvParts.length !== 2) { + return; + } + + const wrapper = csvParts[0]; + const outsideWrapper = entries.some((entry) => { + const parts = entry.split('/').filter(Boolean); + return !isMetadata(parts) && parts[0] !== wrapper; + }); + if (outsideWrapper) { + throw new errors.ValidationError({ message: 'Invalid ZIP file structure.' }); + } +} + +export function isDataFile(fileName: string): boolean { + const parts = fileName.split('/').filter(Boolean); + if (parts.length === 0 || isMetadata(parts)) { + return false; + } + + const extension = extensionOf(fileName); + if (!DATA_EXTENSIONS.has(extension)) { + return false; + } + + if (parts.length === 1) { + return true; + } + + return parts.length === 2; +} + +function extensionOf(fileName: string): string { + return path.posix.extname(fileName).toLowerCase(); +} + +function isUnsupportedCompetingDataFile(fileName: string): boolean { + const extension = extensionOf(fileName); + if (extension === '.csv' || !DATA_EXTENSIONS.has(extension)) { + return false; + } + + const parts = fileName.split('/').filter(Boolean); + if (parts.length === 0 || isMetadata(parts) || isFileAttachment(parts)) { + return false; + } + + return !isDataFile(fileName); +} + +function isFileAttachment(parts: string[]): boolean { + const assetParts = ['content', 'files'].includes(parts[0]) ? [...parts] : parts.slice(1); + if (assetParts[0] === 'content') { + assetParts.shift(); + } + + return assetParts.length > 1 && assetParts[0] === 'files'; +} + +function isMetadata(parts: string[]): boolean { + return ( + parts[0] === '__MACOSX' || parts.some((part) => part.startsWith('._') || part === '.DS_Store') + ); +} diff --git a/ghost/core/core/server/services/content-import/index.ts b/ghost/core/core/server/services/content-import/index.ts index 00f45c117213..313597aea69c 100644 --- a/ghost/core/core/server/services/content-import/index.ts +++ b/ghost/core/core/server/services/content-import/index.ts @@ -1,38 +1,12 @@ import { z } from 'zod'; -import ContentCSVImporter, { - type ImportRequest, - type ImportAccepted, - type FailureReporter, -} from './import/importer'; +import ContentCSVImporter, { type ImportAccepted, type FailureReporter } from './import/importer'; import readPostRows from './import/reader'; -import { EDITORIAL_POST_FIELDS } from './import/row'; +import { importRequestSchema, type ImportRequest } from './import/schema'; import { ImportRunStore } from './import/store'; +import { prepareImportSource } from './import/source'; // The request is built from HTTP upload metadata, so it is validated at the // service boundary rather than trusted. -const importableFields = new Set(EDITORIAL_POST_FIELDS); -const mappingSchema = z.record(z.string(), z.string()).superRefine((mapping, ctx) => { - const targets = new Set(); - for (const [header, target] of Object.entries(mapping)) { - if (!header || header in Object.prototype) { - ctx.addIssue({ code: 'custom', message: `Invalid CSV header mapping: "${header}"` }); - } - if (target && !importableFields.has(target)) { - ctx.addIssue({ code: 'custom', message: `Unknown post field mapping: "${target}"` }); - } - if (target && targets.has(target)) { - ctx.addIssue({ code: 'custom', message: `Post field is mapped more than once: "${target}"` }); - } - targets.add(target); - } - if (!targets.has('title')) { - ctx.addIssue({ code: 'custom', message: 'Post field mapping must include "title"' }); - } -}); -const importRequestSchema = z.object({ - filePath: z.string().min(1), - mapping: mappingSchema.optional(), -}); // A junk timezone setting falls back to UTC rather than mis-stamping the batch tag. const timezoneSchema = z.string().min(1).catch('Etc/UTC'); @@ -68,6 +42,7 @@ function makeImporter(): ContentCSVImporter { return new ContentCSVImporter({ readRows: readPostRows, + prepareSource: prepareImportSource, posts: { create: (data, options) => models.Post.add(data, options), }, diff --git a/ghost/core/core/server/services/email-analytics/lib/queries.ts b/ghost/core/core/server/services/email-analytics/lib/queries.ts index d49c9c38d295..3e2597f5c22e 100644 --- a/ghost/core/core/server/services/email-analytics/lib/queries.ts +++ b/ghost/core/core/server/services/email-analytics/lib/queries.ts @@ -156,7 +156,13 @@ export class Queries { */ async getLastJobRunTimestamp(jobName: EmailAnalyticsJobName): Promise { const jobData = await this.getJobData(jobName); - return jobData ? jobData.finished_at || jobData.started_at : null; + const timestamp = jobData?.finished_at ?? jobData?.started_at ?? null; + if (timestamp === null) { + return null; + } + // SQLite returns datetime columns as strings or numbers, so normalise to a Date to match + // the declared return type and keep callers using Date methods (e.g. getTime()) safe. + return timestamp instanceof Date ? timestamp : new Date(timestamp); } /** diff --git a/ghost/core/core/server/web/api/middleware/upload.js b/ghost/core/core/server/web/api/middleware/upload.js index 958804868b09..6fe826fcca90 100644 --- a/ghost/core/core/server/web/api/middleware/upload.js +++ b/ghost/core/core/server/web/api/middleware/upload.js @@ -37,8 +37,8 @@ const messages = { invalidFile: 'Please select a valid CSV file.', }, posts: { - missingFile: 'Please select a posts CSV file.', - invalidFile: 'Please select a valid CSV file.', + missingFile: 'Please select a posts CSV or ZIP file.', + invalidFile: 'Please select a valid CSV or ZIP file.', }, images: { missingFile: 'Please select an image.', diff --git a/ghost/core/core/shared/config/overrides.json b/ghost/core/core/shared/config/overrides.json index a7400784855b..04a9e8d075ed 100644 --- a/ghost/core/core/shared/config/overrides.json +++ b/ghost/core/core/shared/config/overrides.json @@ -28,12 +28,14 @@ ] }, "posts": { - "extensions": [".csv"], + "extensions": [".csv", ".zip"], "contentTypes": [ "text/csv", "application/csv", "application/octet-stream", - "application/vnd.ms-excel" + "application/vnd.ms-excel", + "application/zip", + "application/x-zip-compressed" ] }, "images": { diff --git a/ghost/core/package.json b/ghost/core/package.json index ff592c701560..06cc38ab2780 100644 --- a/ghost/core/package.json +++ b/ghost/core/package.json @@ -1,406 +1,405 @@ { - "name": "ghost", - "version": "6.60.1-rc.0", - "description": "The professional publishing platform", - "keywords": [ - "blog", - "cms", - "content", - "ghost", - "headless", - "markdown" + "name": "ghost", + "version": "6.60.1-rc.0", + "description": "The professional publishing platform", + "keywords": [ + "blog", + "cms", + "content", + "ghost", + "headless", + "markdown" + ], + "homepage": "https://ghost.org", + "bugs": "https://github.com/TryGhost/Ghost/issues", + "license": "MIT", + "author": "Ghost Foundation", + "contributors": "https://github.com/TryGhost/Ghost/graphs/contributors", + "repository": { + "type": "git", + "url": "git://github.com/TryGhost/Ghost.git" + }, + "files": [ + "core/", + "!core/built/**/*.map", + "!core/built/**/test-*", + "!core/built/**/tests-*", + "content/*/README.md", + "content/themes/casper/", + "!content/themes/casper/pnpm-lock.yaml", + "content/themes/source/", + "!content/themes/source/pnpm-lock.yaml", + "bin/", + "scripts/prune.mts", + "types/", + "index.js", + "loggingrc.js", + "MigratorConfig.js", + "nodemon.json", + "tsconfig.json" + ], + "publishConfig": { + "access": "public", + "registry": "https://registry.npmjs.org/" + }, + "scripts": { + "archive": "node scripts/pack.mjs", + "dev": "nodemon index.js", + "build:assets": "pnpm run '/^build:assets:/'", + "build:assets:js": "node scripts/minify-assets.mjs", + "build:assets:cards": "node scripts/build-card-assets.mjs", + "generate-golden-email": "node bin/generate-golden-email.js", + "migrate:create": "node bin/create-migration.js", + "build:assets:css": "postcss core/frontend/public/ghost.css --no-map --use cssnano -o core/frontend/public/ghost.min.css", + "build:tsc": "tsc", + "pretest": "pnpm build:assets", + "test": "pnpm test:unit", + "test:watch": "vitest --reporter=default", + "test:single": "f() { case \"$1\" in test/unit/*) vitest run \"$1\" ;; test/*) vitest run -c vitest.config.db.ts \"$1\" ;; *) vitest run \"$1\" || vitest run -c vitest.config.db.ts \"$1\" ;; esac; }; f", + "test:all": "pnpm test:unit && pnpm test:integration && pnpm test:e2e && pnpm lint", + "test:debug": "DEBUG=ghost:test* pnpm test", + "test:unit": "vitest run", + "test:integration": "vitest run -c vitest.config.db.ts --project integration", + "test:e2e": "vitest run -c vitest.config.db.ts --project e2e --project e2e-api --project e2e-isolated", + "test:legacy": "vitest run -c vitest.config.db.ts --project legacy", + "test:ci:e2e": "c8 -c ./.c8rc.e2e.json -o coverage-e2e pnpm test:e2e", + "test:ci:integration": "c8 -c ./.c8rc.e2e.json -o coverage-integration --lines 52 --functions 47 --branches 73 --statements 52 pnpm test:integration", + "test:int:slow": "pnpm test:integration", + "test:e2e:slow": "vitest run -c vitest.config.db.ts --project e2e --project e2e-api --reporter=verbose", + "test:leg:slow": "vitest run -c vitest.config.db.ts --project legacy --reporter=verbose", + "lint:server": "eslint 'core/server/**/*.js' 'core/*.js' '*.js' --cache", + "lint:shared": "eslint 'core/shared/**/*.js' --cache", + "lint:frontend": "eslint 'core/frontend/**/*.js' --cache", + "lint:test": "eslint 'test/**/*.js' --cache", + "lint:code": "pnpm run '/^lint:(server|shared|frontend)$/'", + "lint:types": "eslint '**/*.ts' --cache && tsc --noEmit", + "lint": "pnpm run '/^lint:(server|shared|frontend|test|types)$/'" + }, + "dependencies": { + "@aws-sdk/client-s3": "3.1079.0", + "@breejs/later": "catalog:", + "@extractus/oembed-extractor": "3.2.1", + "@faker-js/faker": "catalog:", + "@isaacs/ttlcache": "1.4.1", + "@sentry/node": "7.120.4", + "@slack/webhook": "7.1.0", + "@tryghost/adapter-base-cache": "workspace:*", + "@tryghost/adapter-base-jobs": "workspace:*", + "@tryghost/adapter-base-redirects": "workspace:*", + "@tryghost/adapter-base-route-settings": "workspace:*", + "@tryghost/adapter-base-scheduling": "workspace:*", + "@tryghost/adapter-base-sso": "workspace:*", + "@tryghost/admin-api-schema": "workspace:*", + "@tryghost/api-framework": "catalog:", + "@tryghost/bookshelf-plugins": "2.3.8", + "@tryghost/brute-knex": "catalog:", + "@tryghost/color-utils": "catalog:", + "@tryghost/config-url-helpers": "1.0.27", + "@tryghost/custom-field-types": "workspace:*", + "@tryghost/custom-fonts": "catalog:", + "@tryghost/database-info": "2.3.2", + "@tryghost/debug": "catalog:", + "@tryghost/domain-events": "catalog:", + "@tryghost/email-mock-receiver": "2.1.0", + "@tryghost/errors": "catalog:", + "@tryghost/helpers": "catalog:", + "@tryghost/html-to-plaintext": "1.0.11", + "@tryghost/i18n": "workspace:*", + "@tryghost/image-transform": "1.4.17", + "@tryghost/job-manager": "1.0.9", + "@tryghost/kg-card-factory": "workspace:*", + "@tryghost/kg-converters": "workspace:*", + "@tryghost/kg-default-cards": "workspace:*", + "@tryghost/kg-default-nodes": "workspace:*", + "@tryghost/kg-html-to-lexical": "workspace:*", + "@tryghost/kg-lexical-html-renderer": "workspace:*", + "@tryghost/kg-markdown-html-renderer": "workspace:*", + "@tryghost/limit-service": "catalog:", + "@tryghost/logging": "catalog:", + "@tryghost/metrics": "catalog:", + "@tryghost/mg-clean-html": "catalog:", + "@tryghost/mongo-knex": "catalog:", + "@tryghost/mongo-utils": "0.6.5", + "@tryghost/mw-error-handler": "1.0.13", + "@tryghost/mw-vhost": "1.0.6", + "@tryghost/nodemailer": "2.3.1", + "@tryghost/nql": "catalog:", + "@tryghost/nql-lang": "catalog:", + "@tryghost/nql-string": "workspace:*", + "@tryghost/parse-email-address": "workspace:*", + "@tryghost/pretty-cli": "3.3.2", + "@tryghost/prometheus-metrics": "1.0.8", + "@tryghost/referrer-parser": "0.1.21", + "@tryghost/request": "catalog:", + "@tryghost/root-utils": "2.3.2", + "@tryghost/security": "1.0.6", + "@tryghost/social-urls": "0.1.63", + "@tryghost/string": "catalog:", + "@tryghost/tpl": "catalog:", + "@tryghost/url-utils": "5.2.6", + "@tryghost/validator": "0.2.22", + "@tryghost/version": "2.3.2", + "@tryghost/zip": "3.5.1", + "@x402/core": "catalog:", + "@x402/evm": "catalog:", + "@x402/hono": "catalog:", + "archiver": "8.0.0", + "body-parser": "1.20.6", + "bookshelf": "1.2.0", + "bookshelf-relations": "2.8.0", + "bson-objectid": "catalog:", + "cache-manager": "4.1.0", + "cache-manager-ioredis": "2.1.0", + "chalk": "4.1.2", + "charset": "1.0.1", + "cheerio": "0.22.0", + "clsx": "catalog:", + "common-tags": "1.8.2", + "compression": "1.8.1", + "connect-slashes": "1.4.0", + "cookie-session": "2.1.1", + "cookies": "0.9.1", + "cors": "2.8.6", + "countries-and-timezones": "3.9.0", + "cron-validate": "catalog:", + "csso": "5.0.5", + "dequal": "catalog:", + "dompurify": "catalog:", + "downsize-cjs": "1.1.2", + "entities": "4.5.0", + "express": "4.22.2", + "express-brute": "1.0.1", + "express-hbs": "2.5.0", + "express-jwt": "8.5.1", + "express-lazy-router": "1.0.6", + "express-query-boolean": "2.0.0", + "express-queue": "0.0.13", + "express-session": "1.19.0", + "fastq": "catalog:", + "file-type": "21.3.4", + "form-data": "catalog:", + "fs-extra": "catalog:", + "ghost-storage-base": "workspace:*", + "glob": "catalog:", + "got": "catalog:", + "gscan": "6.4.2", + "handlebars": "4.7.9", + "heic-convert": "2.1.0", + "hono": "catalog:", + "html-to-text": "5.1.1", + "html5parser": "2.0.2", + "human-number": "3.0.0", + "iconv-lite": "0.7.2", + "image-size": "1.2.1", + "intl-messageformat": "5.4.3", + "js-yaml": "catalog:", + "jsdom": "catalog:", + "jsonc-parser": "catalog:", + "jsonwebtoken": "8.5.1", + "juice": "9.1.0", + "knex": "2.4.2", + "knex-migrator": "catalog:", + "leaky-bucket": "2.2.0", + "lodash": "catalog:", + "luxon": "3.7.2", + "mailgun.js": "10.4.0", + "metascraper": "5.55.2", + "metascraper-amazon": "5.55.2", + "metascraper-author": "5.55.2", + "metascraper-description": "5.55.2", + "metascraper-image": "5.55.2", + "metascraper-logo": "5.55.2", + "metascraper-logo-favicon": "5.55.2", + "metascraper-publisher": "5.55.2", + "metascraper-title": "5.55.2", + "metascraper-url": "5.55.2", + "mime-types": "2.1.35", + "mingo": "catalog:", + "moment": "2.24.0", + "moment-timezone": "0.5.45", + "mppx": "catalog:", + "multer": "2.2.0", + "mysql2": "3.22.5", + "nconf": "0.13.0", + "node-html-markdown": "catalog:", + "node-jose": "2.2.0", + "nodemailer": "8.0.11", + "on-headers": "1.1.0", + "otplib": "12.0.1", + "papaparse": "5.5.4", + "path-match": "1.2.4", + "probability-distributions": "0.9.1", + "probe-image-size": "7.3.0", + "rss": "1.2.2", + "sanitize-html": "2.17.5", + "semver": "catalog:", + "sharp": "0.35.3", + "simple-dom": "1.4.0", + "stoppable": "1.1.0", + "stripe": "8.222.0", + "temporal-polyfill": "catalog:", + "terser": "5.48.0", + "tiny-glob": "0.2.9", + "tldts": "catalog:", + "type-fest": "catalog:", + "ua-parser-js": "1.0.41", + "unidecode": "catalog:", + "xml": "1.0.1", + "zod": "catalog:" + }, + "devDependencies": { + "@eslint/js": "catalog:", + "@internal/cfg-eslint": "workspace:*", + "@tryghost/express-test": "2.1.0", + "@tryghost/webhook-mock-receiver": "2.1.0", + "@types/bookshelf": "1.2.9", + "@types/common-tags": "1.8.4", + "@types/express": "catalog:", + "@types/express-brute": "catalog:", + "@types/fs-extra": "11.0.4", + "@types/js-yaml": "4.0.9", + "@types/jsdom": "catalog:", + "@types/jsonwebtoken": "9.0.10", + "@types/lodash": "4.17.25", + "@types/lodash-es": "4.17.12", + "@types/luxon": "catalog:", + "@types/mime-types": "3.0.1", + "@types/nconf": "catalog:", + "@types/node": "catalog:", + "@types/node-jose": "1.1.13", + "@types/nodemailer": "8.0.1", + "@types/on-headers": "1.0.4", + "@types/papaparse": "5.5.2", + "@types/sanitize-html": "catalog:", + "@types/semver": "catalog:", + "@types/sinon": "catalog:", + "@types/stoppable": "catalog:", + "@types/supertest": "6.0.3", + "@typescript/native": "catalog:", + "@vitest/coverage-v8": "catalog:", + "c8": "catalog:", + "chai": "catalog:", + "cli-progress": "3.12.0", + "cssnano": "7.1.9", + "esbuild": "catalog:", + "eslint": "catalog:", + "eslint-plugin-ghost": "3.5.0", + "expect": "29.7.0", + "globals": "17.7.0", + "html-validate": "11.5.5", + "inquirer": "8.2.7", + "jwk-to-pem": "2.0.7", + "jwks-rsa": "3.2.2", + "lodash-es": "4.18.1", + "nock": "catalog:", + "nodemon": "3.1.14", + "postcss": "catalog:", + "postcss-cli": "11.0.1", + "qs": "6.15.3", + "sinon": "catalog:", + "supertest": "catalog:", + "tmp": "0.2.7", + "tsx": "catalog:", + "typescript": "catalog:", + "typescript-eslint": "catalog:", + "validator": "catalog:", + "vitest": "catalog:" + }, + "optionalDependencies": { + "better-sqlite3": "catalog:" + }, + "overrides": { + "cron-validate": "1.4.5" + }, + "engines": { + "cli": "^1.29.1", + "node": "^22.23.1" + }, + "nx": { + "tags": [ + "i18n" ], - "homepage": "https://ghost.org", - "bugs": "https://github.com/TryGhost/Ghost/issues", - "license": "MIT", - "author": "Ghost Foundation", - "contributors": "https://github.com/TryGhost/Ghost/graphs/contributors", - "repository": { - "type": "git", - "url": "git://github.com/TryGhost/Ghost.git" - }, - "files": [ - "core/", - "!core/built/**/*.map", - "!core/built/**/test-*", - "!core/built/**/tests-*", - "content/*/README.md", - "content/themes/casper/", - "!content/themes/casper/pnpm-lock.yaml", - "content/themes/source/", - "!content/themes/source/pnpm-lock.yaml", - "bin/", - "scripts/prune.mts", - "types/", - "index.js", - "loggingrc.js", - "MigratorConfig.js", - "nodemon.json", - "tsconfig.json" - ], - "publishConfig": { - "access": "public", - "registry": "https://registry.npmjs.org/" - }, - "scripts": { - "archive": "node scripts/pack.mjs", - "dev": "nodemon index.js", - "build:assets": "pnpm run '/^build:assets:/'", - "build:assets:js": "node scripts/minify-assets.mjs", - "build:assets:cards": "node scripts/build-card-assets.mjs", - "generate-golden-email": "node bin/generate-golden-email.js", - "migrate:create": "node bin/create-migration.js", - "build:assets:css": "postcss core/frontend/public/ghost.css --no-map --use cssnano -o core/frontend/public/ghost.min.css", - "build:tsc": "tsc", - "pretest": "pnpm build:assets", - "test": "pnpm test:unit", - "test:watch": "vitest --reporter=default", - "test:single": "f() { case \"$1\" in test/unit/*) vitest run \"$1\" ;; test/*) vitest run -c vitest.config.db.ts \"$1\" ;; *) vitest run \"$1\" || vitest run -c vitest.config.db.ts \"$1\" ;; esac; }; f", - "test:all": "pnpm test:unit && pnpm test:integration && pnpm test:e2e && pnpm lint", - "test:debug": "DEBUG=ghost:test* pnpm test", - "test:unit": "vitest run", - "test:integration": "vitest run -c vitest.config.db.ts --project integration", - "test:e2e": "vitest run -c vitest.config.db.ts --project e2e --project e2e-api --project e2e-isolated", - "test:legacy": "vitest run -c vitest.config.db.ts --project legacy", - "test:ci:e2e": "c8 -c ./.c8rc.e2e.json -o coverage-e2e pnpm test:e2e", - "test:ci:integration": "c8 -c ./.c8rc.e2e.json -o coverage-integration --lines 52 --functions 47 --branches 73 --statements 52 pnpm test:integration", - "test:int:slow": "pnpm test:integration", - "test:e2e:slow": "vitest run -c vitest.config.db.ts --project e2e --project e2e-api --reporter=verbose", - "test:leg:slow": "vitest run -c vitest.config.db.ts --project legacy --reporter=verbose", - "lint:server": "eslint 'core/server/**/*.js' 'core/*.js' '*.js' --cache", - "lint:shared": "eslint 'core/shared/**/*.js' --cache", - "lint:frontend": "eslint 'core/frontend/**/*.js' --cache", - "lint:test": "eslint 'test/**/*.js' --cache", - "lint:code": "pnpm run '/^lint:(server|shared|frontend)$/'", - "lint:types": "eslint '**/*.ts' --cache && tsc --noEmit", - "lint": "pnpm run '/^lint:(server|shared|frontend|test|types)$/'" - }, - "dependencies": { - "@aws-sdk/client-s3": "3.1079.0", - "@breejs/later": "catalog:", - "@extractus/oembed-extractor": "3.2.1", - "@faker-js/faker": "catalog:", - "@isaacs/ttlcache": "1.4.1", - "@sentry/node": "7.120.4", - "@slack/webhook": "7.1.0", - "@tryghost/adapter-base-cache": "workspace:*", - "@tryghost/adapter-base-jobs": "workspace:*", - "@tryghost/adapter-base-redirects": "workspace:*", - "@tryghost/adapter-base-route-settings": "workspace:*", - "@tryghost/adapter-base-scheduling": "workspace:*", - "@tryghost/adapter-base-sso": "workspace:*", - "@tryghost/admin-api-schema": "workspace:*", - "@tryghost/api-framework": "catalog:", - "@tryghost/bookshelf-plugins": "2.3.8", - "@tryghost/brute-knex": "catalog:", - "@tryghost/color-utils": "catalog:", - "@tryghost/config-url-helpers": "1.0.27", - "@tryghost/custom-field-types": "workspace:*", - "@tryghost/custom-fonts": "catalog:", - "@tryghost/database-info": "2.3.2", - "@tryghost/debug": "catalog:", - "@tryghost/domain-events": "catalog:", - "@tryghost/email-mock-receiver": "2.1.0", - "@tryghost/errors": "catalog:", - "@tryghost/helpers": "catalog:", - "@tryghost/html-to-plaintext": "1.0.11", - "@tryghost/i18n": "workspace:*", - "@tryghost/image-transform": "1.4.17", - "@tryghost/job-manager": "1.0.9", - "@tryghost/kg-card-factory": "workspace:*", - "@tryghost/kg-converters": "workspace:*", - "@tryghost/kg-default-cards": "workspace:*", - "@tryghost/kg-default-nodes": "workspace:*", - "@tryghost/kg-html-to-lexical": "workspace:*", - "@tryghost/kg-lexical-html-renderer": "workspace:*", - "@tryghost/kg-markdown-html-renderer": "workspace:*", - "@tryghost/limit-service": "catalog:", - "@tryghost/logging": "catalog:", - "@tryghost/metrics": "catalog:", - "@tryghost/mg-clean-html": "catalog:", - "@tryghost/mongo-knex": "catalog:", - "@tryghost/mongo-utils": "0.6.5", - "@tryghost/mw-error-handler": "1.0.13", - "@tryghost/mw-vhost": "1.0.6", - "@tryghost/nodemailer": "2.3.1", - "@tryghost/nql": "catalog:", - "@tryghost/nql-lang": "catalog:", - "@tryghost/nql-string": "workspace:*", - "@tryghost/parse-email-address": "workspace:*", - "@tryghost/pretty-cli": "3.3.2", - "@tryghost/prometheus-metrics": "1.0.8", - "@tryghost/promise": "2.3.2", - "@tryghost/referrer-parser": "0.1.21", - "@tryghost/request": "catalog:", - "@tryghost/root-utils": "2.3.2", - "@tryghost/security": "1.0.6", - "@tryghost/social-urls": "0.1.63", - "@tryghost/string": "catalog:", - "@tryghost/tpl": "catalog:", - "@tryghost/url-utils": "5.2.6", - "@tryghost/validator": "0.2.22", - "@tryghost/version": "2.3.2", - "@tryghost/zip": "3.5.1", - "@x402/core": "catalog:", - "@x402/evm": "catalog:", - "@x402/hono": "catalog:", - "archiver": "8.0.0", - "body-parser": "1.20.6", - "bookshelf": "1.2.0", - "bookshelf-relations": "2.8.0", - "bson-objectid": "catalog:", - "cache-manager": "4.1.0", - "cache-manager-ioredis": "2.1.0", - "chalk": "4.1.2", - "charset": "1.0.1", - "cheerio": "0.22.0", - "clsx": "catalog:", - "common-tags": "1.8.2", - "compression": "1.8.1", - "connect-slashes": "1.4.0", - "cookie-session": "2.1.1", - "cookies": "0.9.1", - "cors": "2.8.6", - "countries-and-timezones": "3.9.0", - "cron-validate": "catalog:", - "csso": "5.0.5", - "dequal": "catalog:", - "dompurify": "catalog:", - "downsize-cjs": "1.1.2", - "entities": "4.5.0", - "express": "4.22.2", - "express-brute": "1.0.1", - "express-hbs": "2.5.0", - "express-jwt": "8.5.1", - "express-lazy-router": "1.0.6", - "express-query-boolean": "2.0.0", - "express-queue": "0.0.13", - "express-session": "1.19.0", - "fastq": "catalog:", - "file-type": "21.3.4", - "form-data": "catalog:", - "fs-extra": "catalog:", - "ghost-storage-base": "workspace:*", - "glob": "catalog:", - "got": "catalog:", - "gscan": "6.4.2", - "handlebars": "4.7.9", - "heic-convert": "2.1.0", - "hono": "catalog:", - "html-to-text": "5.1.1", - "html5parser": "2.0.2", - "human-number": "3.0.0", - "iconv-lite": "0.7.2", - "image-size": "1.2.1", - "intl-messageformat": "5.4.3", - "js-yaml": "catalog:", - "jsdom": "catalog:", - "jsonc-parser": "catalog:", - "jsonwebtoken": "8.5.1", - "juice": "9.1.0", - "knex": "2.4.2", - "knex-migrator": "catalog:", - "leaky-bucket": "2.2.0", - "lodash": "catalog:", - "luxon": "3.7.2", - "mailgun.js": "10.4.0", - "metascraper": "5.55.2", - "metascraper-amazon": "5.55.2", - "metascraper-author": "5.55.2", - "metascraper-description": "5.55.2", - "metascraper-image": "5.55.2", - "metascraper-logo": "5.55.2", - "metascraper-logo-favicon": "5.55.2", - "metascraper-publisher": "5.55.2", - "metascraper-title": "5.55.2", - "metascraper-url": "5.55.2", - "mime-types": "2.1.35", - "mingo": "catalog:", - "moment": "2.24.0", - "moment-timezone": "0.5.45", - "mppx": "catalog:", - "multer": "2.2.0", - "mysql2": "3.22.5", - "nconf": "0.13.0", - "node-html-markdown": "catalog:", - "node-jose": "2.2.0", - "nodemailer": "8.0.11", - "on-headers": "1.1.0", - "otplib": "12.0.1", - "papaparse": "5.5.4", - "path-match": "1.2.4", - "probability-distributions": "0.9.1", - "probe-image-size": "7.3.0", - "rss": "1.2.2", - "sanitize-html": "2.17.5", - "semver": "catalog:", - "sharp": "0.35.3", - "simple-dom": "1.4.0", - "stoppable": "1.1.0", - "stripe": "8.222.0", - "temporal-polyfill": "catalog:", - "terser": "5.48.0", - "tiny-glob": "0.2.9", - "tldts": "catalog:", - "type-fest": "catalog:", - "ua-parser-js": "1.0.41", - "unidecode": "catalog:", - "xml": "1.0.1", - "zod": "catalog:" - }, - "devDependencies": { - "@eslint/js": "catalog:", - "@internal/cfg-eslint": "workspace:*", - "@tryghost/express-test": "2.1.0", - "@tryghost/webhook-mock-receiver": "2.1.0", - "@types/bookshelf": "1.2.9", - "@types/common-tags": "1.8.4", - "@types/express": "catalog:", - "@types/express-brute": "catalog:", - "@types/fs-extra": "11.0.4", - "@types/js-yaml": "4.0.9", - "@types/jsdom": "catalog:", - "@types/jsonwebtoken": "9.0.10", - "@types/lodash": "4.17.25", - "@types/lodash-es": "4.17.12", - "@types/luxon": "catalog:", - "@types/mime-types": "3.0.1", - "@types/nconf": "catalog:", - "@types/node": "catalog:", - "@types/node-jose": "1.1.13", - "@types/nodemailer": "8.0.1", - "@types/on-headers": "1.0.4", - "@types/papaparse": "5.5.2", - "@types/sanitize-html": "catalog:", - "@types/semver": "catalog:", - "@types/sinon": "catalog:", - "@types/stoppable": "catalog:", - "@types/supertest": "6.0.3", - "@typescript/native": "catalog:", - "@vitest/coverage-v8": "catalog:", - "c8": "catalog:", - "chai": "catalog:", - "cli-progress": "3.12.0", - "cssnano": "7.1.9", - "esbuild": "catalog:", - "eslint": "catalog:", - "eslint-plugin-ghost": "3.5.0", - "expect": "29.7.0", - "globals": "17.7.0", - "html-validate": "11.5.5", - "inquirer": "8.2.7", - "jwk-to-pem": "2.0.7", - "jwks-rsa": "3.2.2", - "lodash-es": "4.18.1", - "nock": "catalog:", - "nodemon": "3.1.14", - "postcss": "catalog:", - "postcss-cli": "11.0.1", - "qs": "6.15.3", - "sinon": "catalog:", - "supertest": "catalog:", - "tmp": "0.2.7", - "tsx": "catalog:", - "typescript": "catalog:", - "typescript-eslint": "catalog:", - "validator": "catalog:", - "vitest": "catalog:" - }, - "optionalDependencies": { - "better-sqlite3": "catalog:" - }, - "overrides": { - "cron-validate": "1.4.5" - }, - "engines": { - "cli": "^1.29.1", - "node": "^22.23.1" - }, - "nx": { - "tags": [ - "i18n" + "targets": { + "build:tsc": { + "dependsOn": [ + "^build" + ] + }, + "archive": { + "dependsOn": [ + "build:assets", + "build:tsc", + { + "projects": [ + "@tryghost/admin" + ], + "target": "build" + } + ] + }, + "dev": { + "dependsOn": [ + "build:assets" + ] + }, + "lint": { + "dependsOn": [ + "^build" + ] + }, + "test:all": { + "dependsOn": [ + "build:assets" + ] + }, + "test:unit": { + "dependsOn": [ + "build:assets", + "^build" + ] + }, + "test:e2e": { + "dependsOn": [ + "build:assets", + "^build" + ] + }, + "test:integration": { + "dependsOn": [ + "build:assets", + "^build" + ] + }, + "test:legacy": { + "dependsOn": [ + "build:assets", + "^build" + ] + }, + "test:ci:e2e": { + "dependsOn": [ + "build:assets", + "^build" + ], + "outputs": [ + "{projectRoot}/coverage-e2e" + ] + }, + "test:ci:legacy": { + "dependsOn": [ + "build:assets", + "^build" + ] + }, + "test:ci:integration": { + "dependsOn": [ + "build:assets", + "^build" ], - "targets": { - "build:tsc": { - "dependsOn": [ - "^build" - ] - }, - "archive": { - "dependsOn": [ - "build:assets", - "build:tsc", - { - "projects": [ - "@tryghost/admin" - ], - "target": "build" - } - ] - }, - "dev": { - "dependsOn": [ - "build:assets" - ] - }, - "lint": { - "dependsOn": [ - "^build" - ] - }, - "test:all": { - "dependsOn": [ - "build:assets" - ] - }, - "test:unit": { - "dependsOn": [ - "build:assets", - "^build" - ] - }, - "test:e2e": { - "dependsOn": [ - "build:assets", - "^build" - ] - }, - "test:integration": { - "dependsOn": [ - "build:assets", - "^build" - ] - }, - "test:legacy": { - "dependsOn": [ - "build:assets", - "^build" - ] - }, - "test:ci:e2e": { - "dependsOn": [ - "build:assets", - "^build" - ], - "outputs": [ - "{projectRoot}/coverage-e2e" - ] - }, - "test:ci:legacy": { - "dependsOn": [ - "build:assets", - "^build" - ] - }, - "test:ci:integration": { - "dependsOn": [ - "build:assets", - "^build" - ], - "outputs": [ - "{projectRoot}/coverage-integration" - ] - } - } + "outputs": [ + "{projectRoot}/coverage-integration" + ] + } } + } } diff --git a/ghost/core/test/e2e-api/admin/posts-importer.test.js b/ghost/core/test/e2e-api/admin/posts-importer.test.js index 5a7ac37819ab..4a7f0657bc32 100644 --- a/ghost/core/test/e2e-api/admin/posts-importer.test.js +++ b/ghost/core/test/e2e-api/admin/posts-importer.test.js @@ -13,11 +13,30 @@ const { cacheInvalidateHeaderNotSet } = assertions; const path = require('path'); const models = require('../../../core/server/models'); const jobsService = require('../../../core/server/services/jobs'); +const adapterManager = require('../../../core/server/services/adapter-manager').default; +const { compress } = require('@tryghost/zip'); +const sinon = require('sinon'); const csvPath = path.join(__dirname, '../../utils/fixtures/csv/valid-posts-import.csv'); // Test CSVs are written inline to a temp dir rather than committed as fixtures. let tmpDir; +const getImportedAssetPaths = () => [ + path.join(adapterManager.getAdapter('storage:images').storagePath, 'csv-zip-photo.jpg'), + path.join(adapterManager.getAdapter('storage:media').storagePath, 'csv-zip-movie.mp4'), + path.join(adapterManager.getAdapter('storage:files').storagePath, 'csv-zip-guide.pdf'), + path.join(adapterManager.getAdapter('storage:files').storagePath, 'attachment.csv'), + path.join(adapterManager.getAdapter('storage:images').storagePath, 'canonical-image.jpg'), + path.join(adapterManager.getAdapter('storage:media').storagePath, 'canonical-media.mp4'), + path.join(adapterManager.getAdapter('storage:files').storagePath, 'canonical-file.pdf'), + path.join(adapterManager.getAdapter('storage:files').storagePath, 'rollback-partial-one.pdf'), + path.join(adapterManager.getAdapter('storage:files').storagePath, 'rollback-partial-two.pdf'), + path.join(adapterManager.getAdapter('storage:images').storagePath, 'rollback-cross-image.jpg'), + path.join(adapterManager.getAdapter('storage:files').storagePath, 'rollback-cross-file.pdf'), + path.join(adapterManager.getAdapter('storage:files').storagePath, 'rollback-incomplete-one.pdf'), + path.join(adapterManager.getAdapter('storage:files').storagePath, 'rollback-incomplete-two.pdf'), + path.join(adapterManager.getAdapter('storage:files').storagePath, 'over-cap-guide.pdf'), +]; const csvFile = async (name, content) => { const filePath = path.join(tmpDir, name); @@ -25,6 +44,19 @@ const csvFile = async (name, content) => { return filePath; }; +const zipFile = async (name, files) => { + const source = path.join(tmpDir, `${name}-contents`); + await fs.mkdir(source, { recursive: true }); + for (const [fileName, content] of Object.entries(files)) { + const filePath = path.join(source, fileName); + await fs.mkdir(path.dirname(filePath), { recursive: true }); + await fs.writeFile(filePath, content); + } + const zipPath = path.join(tmpDir, name); + await compress(source, zipPath); + return zipPath; +}; + describe('Posts Importer API', function () { let agent; @@ -42,13 +74,16 @@ describe('Posts Importer API', function () { // Each test logs in as a different role — reset the login rate limiter // so the repeated logins don't trip spam prevention await resetRateLimits(); + await Promise.all(getImportedAssetPaths().map((filePath) => fs.rm(filePath, { force: true }))); }); afterEach(async function () { // Every accepted upload schedules a background import — drain it so a job // doesn't run on into another test (or another file on this fork's DB) await jobsService.allSettled(); + await Promise.all(getImportedAssetPaths().map((filePath) => fs.rm(filePath, { force: true }))); mockManager.restore(); + sinon.restore(); }); it('Can upload a posts CSV as Owner', async function () { @@ -71,6 +106,340 @@ describe('Posts Importer API', function () { .expect(cacheInvalidateHeaderNotSet()); }); + it('Imports the single mapped CSV inside a ZIP', async function () { + await agent.loginAsOwner(); + + const zipPath = await zipFile('mapped-posts.zip', { + 'export/posts.CSV': 'Headline,Body\nZIP mapping post,

Mapped from ZIP

\n', + 'export/content/files/attachment.csv': 'download,only\n', + }); + const form = new FormData(); + form.append('mapping[Headline]', 'title'); + form.append('mapping[Body]', 'html'); + form.append('postsfile', await fs.readFile(zipPath), { + filename: path.basename(zipPath), + contentType: 'application/zip', + }); + + const { body } = await agent.post('posts/upload/').body(form).expectStatus(202); + assert.equal(body.meta.total, 1); + + await jobsService.allSettled(); + const post = await models.Post.findOne({ title: 'ZIP mapping post', status: 'all' }); + assert.ok(post); + assert.match(post.get('html'), /Mapped from ZIP/); + }); + + it('Stores and rewrites wrapped image, media, and file assets before importing posts', async function () { + await agent.loginAsOwner(); + + const csv = + 'title,html,markdown,feature_image,og_image,twitter_image\n' + + 'ZIP HTML assets,"

MediaFile",,/content/images/csv-zip-photo.jpg,/content/images/csv-zip-photo.jpg,/content/images/csv-zip-photo.jpg\n' + + 'ZIP Markdown assets,,"![Image](/content/images/csv-zip-photo.jpg)\n\n[Media](/content/media/csv-zip-movie.mp4)\n\n[File](/content/files/csv-zip-guide.pdf)",,,\n' + + 'ZIP short asset path,"

",,,,\n'; + const zipPath = await zipFile('posts-with-assets.zip', { + 'export/posts.csv': csv, + 'export/content/images/csv-zip-photo.jpg': 'image bytes', + 'export/content/media/csv-zip-movie.mp4': 'media bytes', + 'export/content/files/csv-zip-guide.pdf': 'file bytes', + }); + + const { body } = await agent + .post('posts/upload/') + .attach('postsfile', zipPath) + .expectStatus(202); + assert.equal(body.meta.total, 3); + await jobsService.allSettled(); + + for (const filePath of getImportedAssetPaths().slice(0, 3)) { + assert.equal(await fs.stat(filePath).then(() => true), true, `${filePath} was stored`); + } + const htmlPost = await models.Post.findOne( + { title: 'ZIP HTML assets', status: 'all' }, + { withRelated: ['posts_meta'] }, + ); + const markdownPost = await models.Post.findOne({ + title: 'ZIP Markdown assets', + status: 'all', + }); + const shortPathPost = await models.Post.findOne({ + title: 'ZIP short asset path', + status: 'all', + }); + assert.ok(htmlPost); + assert.ok(markdownPost); + assert.ok(shortPathPost); + for (const assetPath of [ + '/content/images/csv-zip-photo.jpg', + '/content/media/csv-zip-movie.mp4', + '/content/files/csv-zip-guide.pdf', + ]) { + assert.match(htmlPost.get('html'), new RegExp(assetPath)); + assert.match(markdownPost.get('html'), new RegExp(assetPath)); + } + assert.ok(htmlPost.get('feature_image').endsWith('/content/images/csv-zip-photo.jpg')); + assert.ok( + htmlPost.related('posts_meta').get('og_image').endsWith('/content/images/csv-zip-photo.jpg'), + ); + assert.ok( + htmlPost + .related('posts_meta') + .get('twitter_image') + .endsWith('/content/images/csv-zip-photo.jpg'), + ); + assert.match(shortPathPost.get('html'), /\/content\/images\/csv-zip-photo\.jpg/); + }); + + it('Rewrites canonical asset URLs for top-level ZIP asset directories', async function () { + await agent.loginAsOwner(); + + const csv = + 'title,html,feature_image,og_image,twitter_image\n' + + 'Canonical ZIP assets,"

MediaFile",__GHOST_URL__/content/images/canonical-image.jpg,__GHOST_URL__/content/images/canonical-image.jpg,__GHOST_URL__/content/images/canonical-image.jpg\n'; + const zipPath = await zipFile('posts-with-canonical-assets.zip', { + 'posts.csv': csv, + 'images/canonical-image.jpg': 'image bytes', + 'media/canonical-media.mp4': 'media bytes', + 'files/canonical-file.pdf': 'file bytes', + }); + + const { body } = await agent + .post('posts/upload/') + .attach('postsfile', zipPath) + .expectStatus(202); + assert.equal(body.meta.total, 1); + await jobsService.allSettled(); + + for (const filePath of getImportedAssetPaths().slice(4, 7)) { + assert.equal(await fs.stat(filePath).then(() => true), true, `${filePath} was stored`); + } + const post = await models.Post.findOne( + { title: 'Canonical ZIP assets', status: 'all' }, + { withRelated: ['posts_meta'] }, + ); + assert.ok(post); + for (const assetPath of [ + '/content/images/canonical-image.jpg', + '/content/media/canonical-media.mp4', + '/content/files/canonical-file.pdf', + ]) { + assert.match(post.get('html'), new RegExp(assetPath)); + } + assert.doesNotMatch(post.get('html'), /\/content\/content\//); + assert.ok(post.get('feature_image').endsWith('/content/images/canonical-image.jpg')); + assert.ok( + post.related('posts_meta').get('og_image').endsWith('/content/images/canonical-image.jpg'), + ); + assert.ok( + post + .related('posts_meta') + .get('twitter_image') + .endsWith('/content/images/canonical-image.jpg'), + ); + }); + + it('Creates no posts when ZIP asset storage fails', async function () { + await agent.loginAsOwner(); + sinon + .stub(adapterManager.getAdapter('storage:files'), 'save') + .rejects(new Error('storage unavailable')); + const zipPath = await zipFile('posts-with-failed-assets.zip', { + 'posts.csv': 'title,html\nZIP failed assets,

Must not import

\n', + 'content/files/csv-zip-guide.pdf': 'file bytes', + }); + + await agent.post('posts/upload/').attach('postsfile', zipPath).expectStatus(202); + await jobsService.allSettled(); + + const post = await models.Post.findOne({ title: 'ZIP failed assets', status: 'all' }); + assert.equal(post, null); + }); + + it('Rolls back a stored file when another file in the same group fails', async function () { + await agent.loginAsOwner(); + const fileStorage = adapterManager.getAdapter('storage:files'); + const save = fileStorage.save.bind(fileStorage); + let saveCount = 0; + sinon.stub(fileStorage, 'save').callsFake(async (file, targetDir) => { + saveCount += 1; + if (saveCount === 2) { + throw new Error('second file failed'); + } + return save(file, targetDir); + }); + const zipPath = await zipFile('posts-with-partial-file-failure.zip', { + 'posts.csv': 'title,html\nZIP partial file failure,

Must not import

\n', + 'content/files/rollback-partial-one.pdf': 'first file', + 'content/files/rollback-partial-two.pdf': 'second file', + }); + + await agent.post('posts/upload/').attach('postsfile', zipPath).expectStatus(202); + await jobsService.allSettled(); + + const post = await models.Post.findOne({ title: 'ZIP partial file failure', status: 'all' }); + assert.equal(post, null); + for (const filePath of getImportedAssetPaths().slice(7, 9)) { + await assert.rejects(fs.stat(filePath), { code: 'ENOENT' }); + } + }); + + it('Rolls back a successful asset group when another group fails', async function () { + await agent.loginAsOwner(); + sinon + .stub(adapterManager.getAdapter('storage:files'), 'save') + .rejects(new Error('file storage failed')); + const zipPath = await zipFile('posts-with-cross-group-failure.zip', { + 'posts.csv': 'title,html\nZIP cross-group failure,

Must not import

\n', + 'content/images/rollback-cross-image.jpg': 'image bytes', + 'content/files/rollback-cross-file.pdf': 'file bytes', + }); + + await agent.post('posts/upload/').attach('postsfile', zipPath).expectStatus(202); + await jobsService.allSettled(); + + const post = await models.Post.findOne({ title: 'ZIP cross-group failure', status: 'all' }); + assert.equal(post, null); + for (const filePath of getImportedAssetPaths().slice(9, 11)) { + await assert.rejects(fs.stat(filePath), { code: 'ENOENT' }); + } + }); + + it('Reports incomplete rollback without creating posts', async function () { + await agent.loginAsOwner(); + const fileStorage = adapterManager.getAdapter('storage:files'); + const save = fileStorage.save.bind(fileStorage); + let saveCount = 0; + sinon.stub(fileStorage, 'save').callsFake(async (file, targetDir) => { + saveCount += 1; + if (saveCount === 2) { + throw new Error('second file failed'); + } + return save(file, targetDir); + }); + sinon.stub(fileStorage, 'delete').rejects(new Error('rollback failed')); + const zipPath = await zipFile('posts-with-incomplete-rollback.zip', { + 'posts.csv': 'title,html\nZIP incomplete rollback,

Must not import

\n', + 'content/files/rollback-incomplete-one.pdf': 'first file', + 'content/files/rollback-incomplete-two.pdf': 'second file', + }); + + await agent.post('posts/upload/').attach('postsfile', zipPath).expectStatus(202); + await jobsService.allSettled(); + + const post = await models.Post.findOne({ title: 'ZIP incomplete rollback', status: 'all' }); + assert.equal(post, null); + const storedFiles = await Promise.all( + getImportedAssetPaths() + .slice(11, 13) + .map((filePath) => + fs.stat(filePath).then( + () => true, + () => false, + ), + ), + ); + assert.equal(storedFiles.filter(Boolean).length, 1); + }); + + it('Rejects ZIPs with no data CSV, multiple data CSVs, or mixed data formats', async function () { + await agent.loginAsOwner(); + + const cases = [ + { + name: 'no-data-csv.zip', + files: { + 'ghost-import.json': '{}', + 'content/files/attachment.csv': 'download,only\n', + }, + reason: /must contain one CSV file/, + }, + { + name: 'multiple-data-csv.zip', + files: { + 'one.csv': 'title\nOne\n', + 'two.csv': 'title\nTwo\n', + }, + reason: /only one CSV file/, + }, + { + name: 'mixed-data.zip', + files: { + 'posts.csv': 'title\nMixed\n', + 'posts.json': '{}', + }, + reason: /cannot contain CSV, JSON, or Markdown import files together/, + }, + { + name: 'no-importable-content.zip', + files: { + 'readme.txt': 'Nothing to import', + }, + status: 415, + reason: /did not include any content to import/, + }, + { + name: 'nested-data-csv.zip', + files: { + 'export/data/posts.csv': 'title\nNested\n', + }, + status: 415, + reason: /Invalid zip file structure/, + }, + { + name: 'split-wrapper.zip', + files: { + 'export/posts.csv': 'title\nWrapped\n', + 'content/files/attachment.csv': 'download,only\n', + }, + reason: /Invalid ZIP file structure/, + }, + { + name: 'malformed-csv.zip', + files: { + 'posts.csv': 'title,html\nBroken quote,"

never closed\n', + }, + reason: /could not be parsed as a CSV file/, + }, + { + name: 'deeply-nested-json.zip', + files: { + 'posts.csv': 'title\nCSV post\n', + 'export/2024/ghost.json': '{}', + }, + reason: /cannot contain CSV, JSON, or Markdown import files together/, + }, + { + name: 'deeply-nested-markdown.zip', + files: { + 'posts.csv': 'title\nCSV post\n', + 'export/2024/posts.md': '# Markdown post', + }, + reason: /cannot contain CSV, JSON, or Markdown import files together/, + }, + ]; + + for (const testCase of cases) { + const zipPath = await zipFile(testCase.name, testCase.files); + const { body } = await agent + .post('posts/upload/') + .attach('postsfile', zipPath) + .expectStatus(testCase.status ?? 422); + assert.match(body.errors[0].message, testCase.reason); + } + }); + + it('Rejects a corrupt ZIP before scheduling an import', async function () { + await agent.loginAsOwner(); + + const { body } = await agent + .post('posts/upload/') + .attach('postsfile', path.join(__dirname, '../../utils/fixtures/import/zips/empty.zip')) + .expectStatus(415); + + assert.match(body.errors[0].message, /uploaded zip could not be read/i); + }); + it('Cannot upload a posts CSV as Editor', async function () { await agent.loginAsEditor(); @@ -429,14 +798,14 @@ describe('Posts Importer API', function () { { length: 101 }, (_, i) => `Over cap post ${i + 1},

${i + 1}

,2025-02-01T00:00:00.000Z`, ); - const overCapCsvPath = await csvFile( - 'posts-import-over-cap.csv', - 'title,html,published_at\n' + overCapRows.join('\n') + '\n', - ); + const overCapZipPath = await zipFile('posts-import-over-cap.zip', { + 'posts.csv': 'title,html,published_at\n' + overCapRows.join('\n') + '\n', + 'content/files/over-cap-guide.pdf': 'must not be stored', + }); const { body } = await agent .post('posts/upload/') - .attach('postsfile', overCapCsvPath) + .attach('postsfile', overCapZipPath) .expectStatus(422); assert.match(body.errors[0].message, /more than 100 posts/); @@ -449,6 +818,7 @@ describe('Posts Importer API', function () { limit: 'all', }); assert.equal(posts.length, 0, 'no posts were written from the rejected file'); + await assert.rejects(fs.stat(getImportedAssetPaths()[13]), { code: 'ENOENT' }); }); it('Cannot upload a posts CSV when the csvContentImporter flag is disabled', async function () { diff --git a/ghost/core/test/integration/importer/content-import-source.test.ts b/ghost/core/test/integration/importer/content-import-source.test.ts new file mode 100644 index 000000000000..1dc53e36a514 --- /dev/null +++ b/ghost/core/test/integration/importer/content-import-source.test.ts @@ -0,0 +1,252 @@ +import assert from 'node:assert/strict'; +import fs from 'fs-extra'; +import os from 'node:os'; +import path from 'node:path'; +import { prepareAssetBatch } from '../../../core/server/services/content-import/import/assets'; +import { + isDataFile, + prepareImportSource, +} from '../../../core/server/services/content-import/import/source'; + +const { compress } = require('@tryghost/zip'); + +describe('content import source', function () { + let directory: string; + + beforeEach(async function () { + directory = await fs.mkdtemp(path.join(os.tmpdir(), 'content-import-source-')); + }); + + afterEach(async function () { + await fs.remove(directory); + }); + + async function archive(files: Record, name = 'posts.zip'): Promise { + const contents = path.join(directory, 'contents'); + await fs.ensureDir(contents); + for (const [fileName, value] of Object.entries(files)) { + const filePath = path.join(contents, fileName); + await fs.ensureDir(path.dirname(filePath)); + await fs.writeFile(filePath, value); + } + const zipPath = path.join(directory, name); + await compress(contents, zipPath); + return zipPath; + } + + it('leaves a raw CSV path unchanged', async function () { + const filePath = path.join(directory, 'posts.csv'); + await fs.writeFile(filePath, 'title\nRaw'); + + const source = await prepareImportSource({ filePath, fileName: 'posts.csv' }); + + assert.equal(source.filePath, filePath); + await source.cleanup(); + assert.equal(await fs.pathExists(filePath), true); + }); + + it('classifies only root and single-wrapper import data files', function () { + assert.equal(isDataFile('posts.csv'), true); + assert.equal(isDataFile('export/posts.JSON'), true); + assert.equal(isDataFile('export/posts.markdown'), true); + assert.equal(isDataFile(''), false); + assert.equal(isDataFile('readme.txt'), false); + assert.equal(isDataFile('__MACOSX/posts.csv'), false); + assert.equal(isDataFile('export/._posts.csv'), false); + assert.equal(isDataFile('export/data/posts.csv'), false); + assert.equal(isDataFile('content/files/attachment.csv'), false); + }); + + it('ignores a supported asset group when no importer owns it', async function () { + let loadCalls = 0; + const batch = await prepareAssetBatch( + { + getFiles: () => [ + { name: 'content/images/photo.jpg', path: '/tmp/content/images/photo.jpg' }, + ], + }, + '/tmp', + undefined, + { + handlers: [ + { + type: 'images', + extensions: ['.jpg'], + loadFile: async () => { + loadCalls += 1; + return []; + }, + }, + ], + importers: [], + }, + ); + + assert.equal(batch, undefined); + assert.equal(loadCalls, 0); + }); + + it('extracts one case-insensitive root CSV and cleans up its directory', async function () { + const zipPath = await archive({ 'posts.CSV': 'title\nRoot' }); + + const source = await prepareImportSource({ filePath: zipPath, fileName: 'posts.ZIP' }); + + assert.equal(await fs.readFile(source.filePath, 'utf8'), 'title\nRoot'); + const extractedDirectory = path.dirname(source.filePath); + await source.cleanup(); + await source.cleanup(); + assert.equal(await fs.pathExists(extractedDirectory), false, 'cleanup is idempotent'); + }); + + it('extracts a wrapped CSV and prepares wrapped image, media, and file assets', async function () { + const zipPath = await archive({ + 'export/posts.csv': 'title\nWrapped', + 'export/content/images/photo.jpg': 'image', + 'export/content/media/movie.mp4': 'media', + 'export/content/files/attachment.csv': 'download,only', + 'export/content/files/attachment.json': '{"download":true}', + 'export/content/files/attachment.md': '# Download', + '.DS_Store': 'metadata', + }); + + const source = await prepareImportSource({ filePath: zipPath, fileName: 'posts.zip' }); + + assert.equal(await fs.readFile(source.filePath, 'utf8'), 'title\nWrapped'); + assert.deepEqual(source.assets?.files.map((file) => file.originalPath).sort(), [ + 'content/files/attachment.csv', + 'content/files/attachment.json', + 'content/files/attachment.md', + 'content/images/photo.jpg', + 'content/media/movie.mp4', + ]); + assert.deepEqual(source.assets?.files.map((file) => file.newPath).sort(), [ + '/content/files/attachment.csv', + '/content/files/attachment.json', + '/content/files/attachment.md', + '/content/images/photo.jpg', + '/content/media/movie.mp4', + ]); + await source.cleanup(); + }); + + it('prepares assets from the existing top-level directory form', async function () { + const zipPath = await archive({ + 'posts.csv': 'title\nTop level', + 'images/photo.jpg': 'image', + 'media/movie.mp4': 'media', + 'files/guide.pdf': 'file', + }); + + const source = await prepareImportSource({ filePath: zipPath, fileName: 'posts.zip' }); + + assert.deepEqual(source.assets?.files.map((file) => file.originalPath).sort(), [ + 'files/guide.pdf', + 'images/photo.jpg', + 'media/movie.mp4', + ]); + assert.deepEqual(source.assets?.files.map((file) => file.newPath).sort(), [ + '/content/files/guide.pdf', + '/content/images/photo.jpg', + '/content/media/movie.mp4', + ]); + await source.cleanup(); + }); + + it('allows an import-data directory name as the wrapper directory', async function () { + const zipPath = await archive({ 'content/posts.csv': 'title\nWrapped' }); + + const source = await prepareImportSource({ filePath: zipPath, fileName: 'posts.zip' }); + + assert.equal(await fs.readFile(source.filePath, 'utf8'), 'title\nWrapped'); + await source.cleanup(); + }); + + it('ignores macOS metadata CSV entries', async function () { + const zipPath = await archive({ + 'posts.csv': 'title\nPost', + '__MACOSX/._posts.csv': 'metadata', + }); + + const source = await prepareImportSource({ filePath: zipPath, fileName: 'posts.zip' }); + + assert.equal(await fs.readFile(source.filePath, 'utf8'), 'title\nPost'); + await source.cleanup(); + }); + + it('rejects ZIPs without a data CSV and removes extracted files', async function () { + const zipPath = await archive({ + 'ghost-import.json': '{}', + 'content/files/attachment.csv': 'download,only', + }); + + await assert.rejects( + prepareImportSource({ filePath: zipPath, fileName: 'posts.zip' }), + /must contain one CSV file/, + ); + }); + + it('rejects multiple data CSVs', async function () { + const zipPath = await archive({ + 'one.csv': 'title\nOne', + 'two.csv': 'title\nTwo', + }); + + await assert.rejects( + prepareImportSource({ filePath: zipPath, fileName: 'posts.zip' }), + /only one CSV file/, + ); + }); + + it('rejects a CSV nested more deeply than one wrapper directory', async function () { + const zipPath = await archive({ 'export/data/posts.csv': 'title\nNested' }); + + await assert.rejects( + prepareImportSource({ filePath: zipPath, fileName: 'posts.zip' }), + /Invalid zip file structure/i, + ); + }); + + it('rejects content split between the root and a wrapper directory', async function () { + const zipPath = await archive({ + 'export/posts.csv': 'title\nWrapped', + 'content/files/attachment.csv': 'download,only', + }); + + await assert.rejects( + prepareImportSource({ filePath: zipPath, fileName: 'posts.zip' }), + /Invalid ZIP file structure/, + ); + }); + + it('rejects a data CSV mixed with JSON or Markdown data', async function () { + for (const competingName of [ + 'posts.json', + 'posts.md', + 'posts.markdown', + 'export/2024/ghost.json', + 'export/2024/posts.md', + ]) { + const archiveName = `${competingName.replaceAll('/', '-')}.zip`; + const zipPath = await archive( + { + 'posts.csv': 'title\nPost', + [competingName]: 'competing import', + }, + archiveName, + ); + + await assert.rejects( + prepareImportSource({ filePath: zipPath, fileName: archiveName }), + /cannot contain CSV, JSON, or Markdown import files together/, + ); + await fs.remove(path.join(directory, 'contents')); + } + }); + + it('rejects unsupported source extensions', async function () { + await assert.rejects( + prepareImportSource({ filePath: '/tmp/posts.json', fileName: 'posts.json' }), + /valid CSV or ZIP file/, + ); + }); +}); diff --git a/ghost/core/test/integration/importer/import-archive.test.ts b/ghost/core/test/integration/importer/import-archive.test.ts new file mode 100644 index 000000000000..ffbc8d3f179d --- /dev/null +++ b/ghost/core/test/integration/importer/import-archive.test.ts @@ -0,0 +1,223 @@ +import assert from 'node:assert/strict'; +import os from 'node:os'; +import path from 'node:path'; +import fs from 'fs-extra'; +import sinon from 'sinon'; + +const ImportArchive = require('../../../core/server/data/importer/import-archive').default; +const importManager = require('../../../core/server/data/importer/import-manager'); + +describe('ImportArchive', function () { + let directory: string; + + beforeEach(async function () { + directory = await fs.mkdtemp(path.join(os.tmpdir(), 'import-archive-test-')); + }); + + afterEach(async function () { + sinon.restore(); + await fs.remove(directory); + }); + + function archive(options: { extensions?: string[]; directories?: string[] } = {}) { + return new ImportArchive({ + extensions: options.extensions ?? ['.json'], + directories: options.directories ?? [], + }); + } + + async function write(fileName: string, contents = ''): Promise { + const filePath = path.join(directory, fileName); + await fs.ensureDir(path.dirname(filePath)); + await fs.writeFile(filePath, contents); + } + + it('keeps the legacy import manager archive methods on the shared helper', function () { + const getGlobPattern = sinon.stub(importManager.archive, 'getGlobPattern').returns('glob'); + const getExtensionGlob = sinon + .stub(importManager.archive, 'getExtensionGlob') + .returns('extension glob'); + const getDirectoryGlob = sinon + .stub(importManager.archive, 'getDirectoryGlob') + .returns('directory glob'); + const isValid = sinon.stub(importManager.archive, 'isValid').returns(true); + const getFiles = sinon + .stub(importManager.archive, 'getFiles') + .returns([{ name: 'posts.json' }]); + const getBaseDirectory = sinon + .stub(importManager.archive, 'getBaseDirectory') + .returns('export'); + + assert.equal(importManager.getGlobPattern(['.json']), 'glob'); + assert.equal(importManager.getExtensionGlob(['.json'], 1), 'extension glob'); + assert.equal(importManager.getDirectoryGlob(['content'], 2), 'directory glob'); + assert.equal(importManager.isValidZip('/tmp/export'), true); + assert.deepEqual(importManager.getFilesFromZip({ extensions: ['.json'] }, '/tmp/export'), [ + { name: 'posts.json' }, + ]); + assert.equal(importManager.getBaseDirectory('/tmp/export'), 'export'); + + sinon.assert.calledOnceWithExactly(getGlobPattern, ['.json']); + sinon.assert.calledOnceWithExactly(getExtensionGlob, ['.json'], 1); + sinon.assert.calledOnceWithExactly(getDirectoryGlob, ['content'], 2); + sinon.assert.calledOnceWithExactly(isValid, '/tmp/export'); + sinon.assert.calledOnceWithExactly(getFiles, '/tmp/export', ['.json']); + sinon.assert.calledOnceWithExactly(getBaseDirectory, '/tmp/export'); + }); + + it('builds extension and directory globs for each supported depth', function () { + const subject = archive(); + + assert.equal(subject.getGlobPattern(['.json', '.md']), '+(.json|.md)'); + assert.equal(subject.getExtensionGlob(['.json']), '*+(.json)'); + assert.equal(subject.getExtensionGlob(['.json'], 1), '{*/*,*}+(.json)'); + assert.equal(subject.getExtensionGlob(['.json'], 2), '**/*+(.json)'); + assert.equal(subject.getDirectoryGlob(['content']), '+(content)'); + assert.equal(subject.getDirectoryGlob(['content'], 1), '{*/,}+(content)'); + assert.equal(subject.getDirectoryGlob(['content'], 2), '**/+(content)'); + }); + + it('accepts data at the root or in one wrapper directory', async function () { + const subject = archive(); + + await write('POSTS.JSON'); + assert.equal(subject.isValid(directory), true); + + await fs.emptyDir(directory); + await write('export/posts.json'); + assert.equal(subject.isValid(directory), true); + }); + + it('accepts deeply nested data when a supported directory is present', async function () { + const subject = archive({ directories: ['content'] }); + + await write('export/content/images/photo.jpg'); + await write('export/data/posts.json'); + + assert.equal(subject.isValid(directory), true); + }); + + it('rejects archives without importable content', async function () { + await write('readme.txt'); + + assert.throws(() => archive().isValid(directory), /did not include any content/); + }); + + it('rejects data nested more deeply than the supported structure', async function () { + await write('export/data/posts.json'); + + assert.throws(() => archive().isValid(directory), /Invalid zip file structure/); + }); + + it('finds the base directory and handles root data and content directories', async function () { + const subject = archive({ directories: ['content'] }); + + await write('export/posts.json'); + assert.equal(subject.getBaseDirectory(directory), 'export'); + + await fs.emptyDir(directory); + await write('posts.json'); + assert.equal(subject.getBaseDirectory(directory), undefined); + + await fs.emptyDir(directory); + await fs.ensureDir(path.join(directory, 'content')); + assert.equal(subject.getBaseDirectory(directory), undefined); + }); + + it('rejects reading a base directory when there are no data files', function () { + assert.throws(() => archive().getBaseDirectory(directory), /base directory read failed/); + }); + + it('returns case-insensitive file matches with absolute paths', async function () { + await write('export/POSTS.JSON'); + + assert.deepEqual(archive().getFiles(directory, ['.json']), [ + { + name: 'export/POSTS.JSON', + path: path.join(directory, 'export/POSTS.JSON'), + }, + ]); + }); + + it('extracts files and normalizes their permissions', async function () { + const subject = new ImportArchive( + { extensions: ['.json'], directories: [] }, + { + extract: async (_filePath: string, target: string) => { + await fs.outputFile(path.join(target, 'export/posts.json'), '{}'); + await fs.chmod(path.join(target, 'export/posts.json'), 0o600); + }, + }, + ); + + const extracted = await subject.extract('/tmp/posts.zip'); + try { + const mode = (await fs.stat(path.join(extracted, 'export/posts.json'))).mode & 0o777; + assert.equal(mode, 0o644); + } finally { + await fs.remove(extracted); + } + }); + + it('reports invalid filename encoding and removes partial extraction', async function () { + let extracted = ''; + const subject = new ImportArchive( + { extensions: ['.json'], directories: [] }, + { + extract: async (_filePath: string, target: string) => { + extracted = target; + await fs.outputFile(path.join(target, 'partial.json'), '{}'); + throw new Error('ENAMETOOLONG: invalid filename'); + }, + }, + ); + + await assert.rejects( + subject.extract('/tmp/posts.zip'), + (error: { code?: string; message?: string; context?: string }) => { + assert.equal(error.code, 'INVALID_ZIP_FILE_NAME_ENCODING'); + assert.match(error.message ?? '', /could not be read/); + assert.match(error.context ?? '', /filename was too long/); + return true; + }, + ); + assert.equal(await fs.pathExists(extracted), false); + }); + + for (const message of [ + 'end of central directory record signature not found', + 'invalid comment length', + ]) { + it(`reports a malformed ZIP for ${message}`, async function () { + const subject = new ImportArchive( + { extensions: ['.json'], directories: [] }, + { + extract: async () => { + throw new Error(message); + }, + }, + ); + + await assert.rejects(subject.extract('/tmp/posts.zip'), (error: { code?: string }) => { + assert.equal(error.code, 'INVALID_ZIP_FILE'); + return true; + }); + }); + } + + it('preserves unexpected extraction errors', async function () { + const unexpected = new Error('unexpected extraction error'); + const subject = new ImportArchive( + { extensions: ['.json'], directories: [] }, + { + extract: async () => { + throw unexpected; + }, + }, + ); + + await assert.rejects(subject.extract('/tmp/posts.zip'), (error: unknown) => { + return error === unexpected; + }); + }); +}); diff --git a/ghost/core/test/unit/server/adapters/storage/local-base-storage.test.ts b/ghost/core/test/unit/server/adapters/storage/local-base-storage.test.ts index 1f288ea91796..dc08cfe9c84f 100644 --- a/ghost/core/test/unit/server/adapters/storage/local-base-storage.test.ts +++ b/ghost/core/test/unit/server/adapters/storage/local-base-storage.test.ts @@ -1,5 +1,6 @@ import assert from 'assert/strict'; import path from 'path'; +import os from 'os'; import http from 'http'; import express from 'express'; import sinon from 'sinon'; @@ -121,6 +122,47 @@ describe('Local Storage Base', function () { }); describe('path validation', function () { + it('save allows an explicit storage-root target while keeping the file inside it', async function () { + const storagePath = await fs.mkdtemp(path.join(os.tmpdir(), 'ghost-local-storage-')); + const sourcePath = path.join(storagePath, '..', `ghost-local-storage-source-${Date.now()}`); + await fs.writeFile(sourcePath, 'stored at root'); + + try { + const localStorageBase = new LocalStorageBase({ + storagePath, + staticFileURLPrefix: 'content/files', + siteUrl: 'http://example.com/', + }); + + const url = await localStorageBase.save( + { name: 'root-file.txt', path: sourcePath }, + storagePath, + ); + + assert.equal(url, '/content/files/root-file.txt'); + assert.equal( + await fs.readFile(path.join(storagePath, 'root-file.txt'), 'utf8'), + 'stored at root', + ); + } finally { + await fs.remove(storagePath); + await fs.remove(sourcePath); + } + }); + + it('save still rejects target directories outside the storage root', async function () { + const localStorageBase = new LocalStorageBase({ + storagePath: '/media-storage/path/', + staticFileURLPrefix: 'content/media', + siteUrl: 'http://example.com/blog/', + }); + + await assert.rejects( + localStorageBase.save({ name: 'file.txt', path: '/tmp/file.txt' }, '../../outside-root'), + { message: 'The path "../../outside-root" is not valid for this storage.' }, + ); + }); + it('read rejects if the path resolves outside the storage root', async function () { const localStorageBase = new LocalStorageBase({ storagePath: '/media-storage/path/', diff --git a/ghost/core/test/unit/server/data/importer/importers/content-file-importer.test.js b/ghost/core/test/unit/server/data/importer/importers/content-file-importer.test.js index ae497924b7fe..39d64e6b5a31 100644 --- a/ghost/core/test/unit/server/data/importer/importers/content-file-importer.test.js +++ b/ghost/core/test/unit/server/data/importer/importers/content-file-importer.test.js @@ -17,10 +17,13 @@ describe('ContentFileImporter', function () { assert.equal(imageImporter.type, 'images'); assert.equal(imageImporter.preProcess instanceof Function, true); assert.equal(imageImporter.doImport instanceof Function, true); + assert.equal(imageImporter.rollback instanceof Function, true); }); it('does preprocess posts, users and tags correctly', function () { - let inputData = require('../../../../../utils/fixtures/import/import-data-1.json'); + let inputData = _.cloneDeep(require('../../../../../utils/fixtures/import/import-data-1.json')); + inputData.data.data.posts[0].og_image = '/images/my-image.png'; + inputData.data.data.posts[0].twitter_image = '/images/photos/cat.jpg'; const imageImporter = new ContentFileImporter({ type: 'images', store: {}, @@ -42,6 +45,10 @@ describe('ContentFileImporter', function () { assert.equal(inputData.posts[0].feature_image, '/images/my-image.png'); assert.equal(outputData.posts[0].feature_image, '/content/images/my-image.png'); + assert.equal(inputData.posts[0].og_image, '/images/my-image.png'); + assert.equal(outputData.posts[0].og_image, '/content/images/my-image.png'); + assert.equal(inputData.posts[0].twitter_image, '/images/photos/cat.jpg'); + assert.equal(outputData.posts[0].twitter_image, '/content/images/photos/cat.jpg'); assert.equal(inputData.tags[0].feature_image, '/images/my-image.png'); assert.equal(outputData.tags[0].feature_image, '/content/images/my-image.png'); @@ -52,6 +59,69 @@ describe('ContentFileImporter', function () { assert.equal(outputData.users[0].cover_image, '/content/images/photos/cat.jpg'); }); + it('rewrites canonical asset paths without duplicating the content directory', function () { + const imageImporter = new ContentFileImporter({ + type: 'images', + store: {}, + }); + const inputData = { + images: [ + { + originalPath: 'images/image.jpg', + newPath: '/blog/content/images/image-1.jpg', + }, + ], + data: { + data: { + posts: [ + { + html: '', + markdown: '![Image](/content/images/image.jpg)', + feature_image: '__GHOST_URL__/content/images/image.jpg', + og_image: '/content/images/image.jpg', + twitter_image: '/images/image.jpg', + }, + ], + tags: [], + users: [], + }, + }, + }; + + const output = imageImporter.preProcess(inputData); + const post = output.data.data.posts[0]; + + assert.equal(post.html, ''); + assert.equal(post.markdown, '![Image](/blog/content/images/image-1.jpg)'); + assert.equal(post.feature_image, '/blog/content/images/image-1.jpg'); + assert.equal(post.og_image, '/blog/content/images/image-1.jpg'); + assert.equal(post.twitter_image, '/blog/content/images/image-1.jpg'); + assert.doesNotMatch(JSON.stringify(post), /content\/content/); + }); + + it('rewrites a short post path when the archive path is canonical', function () { + const imageImporter = new ContentFileImporter({ type: 'images', store: {} }); + const inputData = { + images: [ + { + originalPath: 'content/images/image.jpg', + newPath: '/content/images/image-1.jpg', + }, + ], + data: { + data: { + posts: [{ html: '' }], + tags: [], + users: [], + }, + }, + }; + + const output = imageImporter.preProcess(inputData); + + assert.equal(output.data.data.posts[0].html, ''); + }); + it('does import the images correctly', async function () { const inputData = require('../../../../../utils/fixtures/import/import-data-1.json'); const storageApi = { @@ -82,6 +152,74 @@ describe('ContentFileImporter', function () { assert.equal(storageApi.save.args[0][0].newPath, '/content/files/best-memes.pdf'); }); + it('waits for every asset write to settle before reporting a storage failure', async function () { + let finishSecondWrite; + const remove = sinon.stub().resolves(); + const storageApi = { + save: sinon + .stub() + .onFirstCall() + .rejects(new Error('first write failed')) + .onSecondCall() + .returns( + new Promise((resolve) => { + finishSecondWrite = () => resolve('/content/images/two.jpg'); + }), + ), + urlToPath: sinon.stub().returns('two.jpg'), + delete: remove, + }; + const importer = new ContentFileImporter({ type: 'images', store: storageApi }); + const importPromise = importer.doImport([ + { name: 'one.jpg', targetDir: '/images' }, + { name: 'two.jpg', targetDir: '/images' }, + ]); + let rejected = false; + importPromise.catch(() => { + rejected = true; + }); + + await new Promise((resolve) => { + setImmediate(resolve); + }); + assert.equal(rejected, false, 'the batch remains active while another write is pending'); + finishSecondWrite(); + + await assert.rejects(importPromise, /first write failed/); + sinon.assert.calledTwice(storageApi.save); + sinon.assert.calledOnceWithExactly(remove, 'two.jpg', undefined); + }); + + it('reports when a partially stored batch cannot be completely rolled back', async function () { + const storageFailure = new Error('first write failed'); + const rollbackFailure = new Error('delete failed'); + const storageApi = { + save: sinon + .stub() + .onFirstCall() + .rejects(storageFailure) + .onSecondCall() + .resolves('/content/images/two.jpg'), + urlToPath: sinon.stub().returns('nested/two.jpg'), + delete: sinon.stub().rejects(rollbackFailure), + }; + const importer = new ContentFileImporter({ type: 'images', store: storageApi }); + + await assert.rejects( + importer.doImport([ + { name: 'one.jpg', targetDir: '/images' }, + { name: 'two.jpg', targetDir: '/images' }, + ]), + (error) => { + assert.equal(error.message, 'Asset storage failed and rollback was incomplete.'); + assert.equal(error.cause, storageFailure); + assert.deepEqual(error.errors, [storageFailure, rollbackFailure]); + return true; + }, + ); + sinon.assert.calledOnceWithExactly(storageApi.delete, 'two.jpg', 'nested'); + }); + describe('doImport with CDN storage', function () { it('stores CDN URL returned by save() in the result.stored field', async function () { const images = [ diff --git a/ghost/core/test/unit/server/lib/promise-pool.test.ts b/ghost/core/test/unit/server/lib/promise-pool.test.ts new file mode 100644 index 000000000000..6760ae9e0b20 --- /dev/null +++ b/ghost/core/test/unit/server/lib/promise-pool.test.ts @@ -0,0 +1,44 @@ +import { times } from 'lodash'; +import assert from 'node:assert/strict'; +import { setImmediate } from 'node:timers/promises'; +import * as sinon from 'sinon'; +import { promisePool } from '../../../../core/server/lib/promise-pool'; + +describe('promisePool', function () { + it('rejects with invalid concurrency', async function () { + for (const maxConcurrency of [-1, 0, 1.5, Number.MAX_SAFE_INTEGER + 1, NaN, Infinity]) { + await assert.rejects(promisePool([], maxConcurrency), { + message: 'Concurrency must be a positive integer', + }); + } + }); + + it('limits concurrency', async function () { + let running = 0; + let maxRunning = 0; + + await promisePool( + times(100, () => async () => { + running += 1; + maxRunning = Math.max(maxRunning, running); + await setImmediate(); + running -= 1; + }), + 3, + ); + + assert.equal(maxRunning, 3); + }); + + it('rejects if a task fails', async function () { + const finalTask = sinon.stub().resolves(); + const tasks = [ + () => Promise.resolve(), + () => Promise.reject(new Error('Task failed')), + finalTask, + ]; + + await assert.rejects(promisePool(tasks, 1), { message: 'Task failed' }); + sinon.assert.notCalled(finalTask); + }); +}); diff --git a/ghost/core/test/unit/server/services/content-import/import/assets.test.ts b/ghost/core/test/unit/server/services/content-import/import/assets.test.ts new file mode 100644 index 000000000000..164a3948e730 --- /dev/null +++ b/ghost/core/test/unit/server/services/content-import/import/assets.test.ts @@ -0,0 +1,229 @@ +import assert from 'node:assert/strict'; +import sinon from 'sinon'; +import { + belongsToAssetDirectory, + prepareAssetBatch, + PreparedAssetBatch, +} from '../../../../../../core/server/services/content-import/import/assets'; +import type { PostImportRow } from '../../../../../../core/server/services/content-import/import/row'; + +const ContentFileImporter = require('../../../../../../core/server/data/importer/importers/content-file-importer'); + +describe('content import assets', function () { + afterEach(function () { + sinon.restore(); + }); + + it('recognizes top-level, content, and wrapper asset directories', function () { + assert.equal(belongsToAssetDirectory('images/photo.jpg', 'images'), true); + assert.equal(belongsToAssetDirectory('content/images/photo.jpg', 'images'), true); + assert.equal( + belongsToAssetDirectory('export/content/images/photo.jpg', 'images', 'export'), + true, + ); + assert.equal(belongsToAssetDirectory('export/media/movie.mp4', 'media', 'export'), true); + assert.equal( + belongsToAssetDirectory('export/content/files/guide.pdf', 'files', 'export'), + true, + ); + assert.equal(belongsToAssetDirectory('photo.jpg', 'images'), false); + assert.equal(belongsToAssetDirectory('content/files/photo.jpg', 'images'), false); + }); + + it('prepares, stores, and rewrites assets with the existing content-file importer', async function () { + const save = sinon.stub().resolves('/content/images/unique.jpg'); + const handler = { + type: 'images' as const, + extensions: ['.jpg'], + loadFile: sinon.stub().callsFake(async (files: Array<{ name: string; path: string }>) => + files.map((file) => ({ + ...file, + name: 'unique.jpg', + originalPath: 'content/images/original.jpg', + newPath: '/blog/content/images/unique.jpg', + targetDir: '/var/lib/ghost/content/images', + })), + ), + }; + const importer = new ContentFileImporter({ type: 'images', store: { save } }); + const archive = { + getFiles: sinon.stub().returns([ + { + name: 'export/content/images/original.jpg', + path: '/tmp/export/content/images/original.jpg', + }, + { + name: 'export/content/files/not-an-image.jpg', + path: '/tmp/export/content/files/not-an-image.jpg', + }, + ]), + }; + + const batch = await prepareAssetBatch(archive, '/tmp/export', 'export', { + handlers: [handler], + importers: [importer], + }); + + assert.ok(batch); + assert.equal(batch.files.length, 1); + assert.equal(batch.files[0].newPath, '/blog/content/images/unique.jpg'); + await batch.store(); + sinon.assert.calledOnceWithExactly( + save, + sinon.match({ name: 'unique.jpg', originalPath: 'content/images/original.jpg' }), + '/var/lib/ghost/content/images', + ); + + const rows: PostImportRow[] = [ + { + title: 'Assets', + html: '', + markdown: '![Image](content/images/original.jpg)', + feature_image: '/content/images/original.jpg', + og_image: 'content/images/original.jpg', + twitter_image: '/content/images/original.jpg', + }, + ]; + batch.rewriteRows(rows); + + assert.match(rows[0].html, /\/blog\/content\/images\/unique\.jpg/); + assert.match(rows[0].markdown, /\/blog\/content\/images\/unique\.jpg/); + assert.equal(rows[0].feature_image, '/blog/content/images/unique.jpg'); + assert.equal(rows[0].og_image, '/blog/content/images/unique.jpg'); + assert.equal(rows[0].twitter_image, '/blog/content/images/unique.jpg'); + }); + + it('returns no batch when the archive has no supported asset directories', async function () { + const loadFile = sinon.stub(); + const batch = await prepareAssetBatch( + { getFiles: () => [{ name: 'root.jpg', path: '/tmp/root.jpg' }] }, + '/tmp', + undefined, + { + handlers: [{ type: 'images', extensions: ['.jpg'], loadFile }], + importers: [], + }, + ); + + assert.equal(batch, undefined); + sinon.assert.notCalled(loadFile); + }); + + it('waits for every asset group to settle before reporting a storage failure', async function () { + let finishMediaWrite: () => void = () => {}; + const failedWrite = new Error('image storage failed'); + const batch = new PreparedAssetBatch([ + { + type: 'images', + files: [], + importer: { + type: 'images', + doImport: async () => { + throw failedWrite; + }, + rollback: sinon.stub().resolves(), + preProcess: (data: Record) => data, + }, + }, + { + type: 'media', + files: [], + importer: { + type: 'media', + doImport: () => + new Promise<[]>((resolve) => { + finishMediaWrite = () => resolve([]); + }), + rollback: sinon.stub().resolves(), + preProcess: (data: Record) => data, + }, + }, + ]); + let rejected = false; + const storePromise = batch.store().catch((error) => { + rejected = true; + throw error; + }); + + await new Promise((resolve) => { + setImmediate(resolve); + }); + assert.equal(rejected, false); + + finishMediaWrite(); + await assert.rejects(storePromise, /image storage failed/); + }); + + it('rolls back successful asset groups when another group fails', async function () { + const failedWrite = new Error('image storage failed'); + const storedMedia = { + originalPath: 'content/media/movie.mp4', + newPath: '/content/media/movie.mp4', + stored: '/content/media/movie.mp4', + }; + const rollbackMedia = sinon.stub().resolves(); + const batch = new PreparedAssetBatch([ + { + type: 'images', + files: [], + importer: { + type: 'images', + doImport: async () => { + throw failedWrite; + }, + rollback: sinon.stub().resolves(), + preProcess: (data: Record) => data, + }, + }, + { + type: 'media', + files: [], + importer: { + type: 'media', + doImport: async () => [storedMedia], + rollback: rollbackMedia, + preProcess: (data: Record) => data, + }, + }, + ]); + + await assert.rejects(batch.store(), (error: unknown) => error === failedWrite); + sinon.assert.calledOnceWithExactly(rollbackMedia, [storedMedia]); + }); + + it('preserves storage and rollback failures when rollback is incomplete', async function () { + const failedWrite = new Error('image storage failed'); + const failedRollback = new Error('media rollback failed'); + const batch = new PreparedAssetBatch([ + { + type: 'images', + files: [], + importer: { + type: 'images', + doImport: async () => { + throw failedWrite; + }, + rollback: sinon.stub().resolves(), + preProcess: (data: Record) => data, + }, + }, + { + type: 'media', + files: [], + importer: { + type: 'media', + doImport: async () => [], + rollback: sinon.stub().rejects(failedRollback), + preProcess: (data: Record) => data, + }, + }, + ]); + + await assert.rejects(batch.store(), (error: unknown) => { + assert.ok(error instanceof AggregateError); + assert.equal(error.cause, failedWrite); + assert.deepEqual(error.errors, [failedWrite, failedRollback]); + return true; + }); + }); +}); diff --git a/ghost/core/test/unit/server/services/content-import/import/importer.test.ts b/ghost/core/test/unit/server/services/content-import/import/importer.test.ts index 16b4a5720e7d..d08124bf46d7 100644 --- a/ghost/core/test/unit/server/services/content-import/import/importer.test.ts +++ b/ghost/core/test/unit/server/services/content-import/import/importer.test.ts @@ -77,7 +77,10 @@ function harness(rows: PostImportRow[] = [row('First'), row('Second')]) { // The scheduled job is invoked directly rather than through the job manager. const run = async () => { - const accepted = await importer.importCSV({ filePath: '/tmp/posts.csv' }); + const accepted = await importer.importCSV({ + filePath: '/tmp/posts.csv', + fileName: 'posts.csv', + }); for (const job of jobs) { await job.job(); } @@ -129,7 +132,10 @@ describe('ContentCSVImporter', function () { it('accepts the upload with the row count and defers the writes to one inline job', async function () { const h = harness(); - const accepted = await h.importer.importCSV({ filePath: '/tmp/posts.csv' }); + const accepted = await h.importer.importCSV({ + filePath: '/tmp/posts.csv', + fileName: 'posts.csv', + }); assert.deepEqual(accepted, { importId: 'run_test', total: 2 }); assert.equal(h.jobs.length, 1); @@ -153,6 +159,18 @@ describe('ContentCSVImporter', function () { sinon.assert.calledWithMatch(infoLog, /^\[Background Job\] content-import completed in \d+ms$/); }); + it('uses the current time by default when one is not injected', async function () { + const h = harness([row('Default clock')]); + const { now: _now, ...deps } = h.deps; + const importer = new ContentCSVImporter(deps); + + await importer.importCSV({ filePath: '/tmp/posts.csv', fileName: 'posts.csv' }); + await h.jobs[0].job(); + + assert.ok(h.store.get('run_test')?.startedAt instanceof Date); + assert.ok(h.store.get('run_test')?.finishedAt instanceof Date); + }); + it('keeps lifecycle logging best-effort', async function () { const h = harness(); infoLog.throws(new Error('Logging unavailable')); @@ -180,11 +198,103 @@ describe('ContentCSVImporter', function () { }, }); - await importer.importCSV({ filePath: '/tmp/posts.csv', mapping: { Headline: 'title' } }); + await importer.importCSV({ + filePath: '/tmp/posts.csv', + fileName: 'posts.csv', + mapping: { Headline: 'title' }, + }); assert.deepEqual(receivedMapping, { Headline: 'title' }); }); + it('reads a prepared archive source and cleans it after the job', async function () { + const h = harness(); + const cleanup = sinon.stub().resolves(); + let receivedPath: string | undefined; + const importer = new ContentCSVImporter({ + ...h.deps, + prepareSource: async () => ({ filePath: '/tmp/extracted/posts.csv', cleanup }), + readRows: async (filePath) => { + receivedPath = filePath; + return [row('Prepared')]; + }, + }); + + await importer.importCSV({ filePath: '/tmp/upload', fileName: 'posts.zip' }); + + assert.equal(receivedPath, '/tmp/extracted/posts.csv'); + sinon.assert.notCalled(cleanup); + await h.jobs[0].job(); + sinon.assert.calledOnce(cleanup); + }); + + it('stores and rewrites every asset before resolving content converters', async function () { + const h = harness([row('Prepared', '')]); + const events: string[] = []; + const cleanup = sinon.stub().resolves(); + const assets = { + files: [], + store: async () => { + events.push('store'); + }, + rewriteRows: (rows: PostImportRow[]) => { + events.push('rewrite'); + rows[0].html = ''; + }, + }; + h.setHtmlToLexicalFactory(() => { + events.push('convert'); + return (html: string) => ({ converted: html }); + }); + const importer = new ContentCSVImporter({ + ...h.deps, + prepareSource: async () => ({ + filePath: '/tmp/extracted/posts.csv', + assets, + cleanup, + }), + }); + + await importer.importCSV({ filePath: '/tmp/posts.zip', fileName: 'posts.zip' }); + await h.jobs[0].job(); + + assert.deepEqual(events.slice(0, 3), ['store', 'rewrite', 'convert']); + assert.match(h.created[0].data.lexical ?? '', /unique\.jpg/); + sinon.assert.calledOnce(cleanup); + }); + + it('fails the run without converting or creating posts when asset storage fails', async function () { + const h = harness(); + const failure = new Error('storage unavailable'); + const cleanup = sinon.stub().resolves(); + const rewriteRows = sinon.stub(); + const importer = new ContentCSVImporter({ + ...h.deps, + prepareSource: async () => ({ + filePath: '/tmp/extracted/posts.csv', + assets: { + files: [], + store: async () => { + throw failure; + }, + rewriteRows, + }, + cleanup, + }), + }); + + await importer.importCSV({ filePath: '/tmp/posts.zip', fileName: 'posts.zip' }); + await h.jobs[0].job(); + + assert.equal(h.converterResolutions(), 0); + assert.equal(h.created.length, 0); + assert.equal(h.store.get('run_test')?.status, 'failed'); + assert.equal(h.store.get('run_test')?.failureReason, 'storage unavailable'); + assert.equal(h.reported.at(-1), failure); + sinon.assert.notCalled(rewriteRows); + sinon.assert.calledOnce(cleanup); + }); + it('writes one post per row, in order, under the importing options', async function () { const h = harness(); @@ -215,15 +325,18 @@ describe('ContentCSVImporter', function () { it('rejects a file that cannot be parsed as CSV without scheduling any work', async function () { const h = harness(); + const cleanupError = new Error('cleanup failed'); + const cleanup = sinon.stub().rejects(cleanupError); const importer = new ContentCSVImporter({ ...h.deps, + prepareSource: async () => ({ filePath: '/tmp/extracted/posts.csv', cleanup }), readRows: async () => { throw new Error('bad bytes'); }, }); await assert.rejects( - importer.importCSV({ filePath: '/tmp/posts.csv' }), + importer.importCSV({ filePath: '/tmp/posts.csv', fileName: 'posts.csv' }), (error: { errorType?: string; message?: string }) => { assert.equal(error.errorType, 'ValidationError'); assert.match(error.message ?? '', /could not be parsed as a CSV/); @@ -232,6 +345,8 @@ describe('ContentCSVImporter', function () { ); assert.equal(h.jobs.length, 0, 'no job was scheduled'); + sinon.assert.calledOnce(cleanup); + assert.equal(h.reported.at(-1), cleanupError); }); it('resolves the html converter once per run, not per row', async function () { @@ -260,9 +375,23 @@ describe('ContentCSVImporter', function () { it('rejects a file over the cap without scheduling any work', async function () { const h = harness(Array.from({ length: 101 }, (_, i) => row(`Post ${i + 1}`))); + const cleanup = sinon.stub().resolves(); + const storeAssets = sinon.stub().resolves(); + const importer = new ContentCSVImporter({ + ...h.deps, + prepareSource: async () => ({ + filePath: '/tmp/extracted/posts.csv', + assets: { + files: [], + store: storeAssets, + rewriteRows: sinon.stub(), + }, + cleanup, + }), + }); await assert.rejects( - h.importer.importCSV({ filePath: '/tmp/posts.csv' }), + importer.importCSV({ filePath: '/tmp/posts.zip', fileName: 'posts.zip' }), (error: { errorType?: string; message?: string }) => { assert.equal(error.errorType, 'ValidationError'); assert.match(error.message ?? '', /more than 100 posts/); @@ -273,12 +402,57 @@ describe('ContentCSVImporter', function () { assert.equal(h.jobs.length, 0, 'no job was scheduled'); assert.equal(h.created.length, 0, 'nothing was written'); assert.equal(h.store.get('run_test'), undefined, 'no run was registered'); + sinon.assert.notCalled(storeAssets); + sinon.assert.calledOnce(cleanup); + }); + + it('cleans a prepared source if scheduling throws', async function () { + const h = harness(); + const cleanup = sinon.stub().resolves(); + const importer = new ContentCSVImporter({ + ...h.deps, + prepareSource: async () => ({ filePath: '/tmp/extracted/posts.csv', cleanup }), + addJob: () => { + throw new Error('queue unavailable'); + }, + }); + + await assert.rejects( + importer.importCSV({ filePath: '/tmp/posts.zip', fileName: 'posts.zip' }), + /queue unavailable/, + ); + assert.equal(h.store.get('run_test')?.status, 'failed'); + assert.equal(h.store.get('run_test')?.failureReason, 'queue unavailable'); + sinon.assert.calledOnce(cleanup); + }); + + it('reports cleanup failures without rejecting the completed job', async function () { + const h = harness(); + const cleanupError = new Error('cleanup failed'); + const importer = new ContentCSVImporter({ + ...h.deps, + prepareSource: async () => ({ + filePath: '/tmp/extracted/posts.csv', + cleanup: async () => { + throw cleanupError; + }, + }), + }); + + await importer.importCSV({ filePath: '/tmp/posts.zip', fileName: 'posts.zip' }); + await h.jobs[0].job(); + + assert.equal(h.store.get('run_test')?.status, 'complete'); + assert.equal(h.reported.at(-1), cleanupError); }); it('accepts a file exactly at the cap', async function () { const h = harness(Array.from({ length: 100 }, (_, i) => row(`Post ${i + 1}`))); - const accepted = await h.importer.importCSV({ filePath: '/tmp/posts.csv' }); + const accepted = await h.importer.importCSV({ + filePath: '/tmp/posts.csv', + fileName: 'posts.csv', + }); assert.deepEqual(accepted, { importId: 'run_test', total: 100 }); assert.equal(h.jobs.length, 1); @@ -440,6 +614,37 @@ describe('ContentCSVImporter', function () { sinon.assert.calledWithMatch(infoLog, /^\[Background Job\] content-import failed after \d+ms$/); }); + it('stops the run for an unexpected row-processing failure', async function () { + const failure = new Error('unexpected row failure'); + const badRow = row('Bad row'); + Object.defineProperty(badRow, 'html', { + get() { + throw failure; + }, + }); + const h = harness([badRow, row('Never reached')]); + + await h.run(); + + assert.deepEqual(h.reported, [failure]); + assert.equal(h.created.length, 0); + assert.equal(h.store.get('run_test')?.status, 'failed'); + assert.equal(h.store.get('run_test')?.failureReason, 'unexpected row failure'); + }); + + it('uses a safe reason when a run-level failure has no message', async function () { + const h = harness(); + h.setHtmlToLexicalFactory(() => { + throw {}; + }); + + await h.run(); + + assert.deepEqual(h.reported, [{}]); + assert.equal(h.store.get('run_test')?.status, 'failed'); + assert.equal(h.store.get('run_test')?.failureReason, 'Unknown error'); + }); + it('keeps a successfully written post created when its URL cannot be resolved', async function () { const h = harness(); h.urlFailures.set('post_1', new Error('URL service unavailable')); diff --git a/ghost/core/test/unit/server/services/content-import/import/schema.test.ts b/ghost/core/test/unit/server/services/content-import/import/schema.test.ts new file mode 100644 index 000000000000..d08044a7defc --- /dev/null +++ b/ghost/core/test/unit/server/services/content-import/import/schema.test.ts @@ -0,0 +1,76 @@ +import assert from 'node:assert/strict'; +import { + importRequestSchema, + mappingSchema, +} from '../../../../../../core/server/services/content-import/import/schema'; + +function issuesFor(mapping: Record): string[] { + const result = mappingSchema.safeParse(mapping); + if (result.success) { + assert.fail('Expected mapping validation to fail'); + } + return result.error.issues.map((issue) => issue.message); +} + +describe('content import schema', function () { + it('accepts a CSV or ZIP request with a valid editable mapping', function () { + assert.deepEqual( + importRequestSchema.parse({ + filePath: '/tmp/posts.zip', + fileName: 'posts.zip', + mapping: { + Headline: 'title', + Body: 'html', + Notes: '', + }, + }), + { + filePath: '/tmp/posts.zip', + fileName: 'posts.zip', + mapping: { + Headline: 'title', + Body: 'html', + Notes: '', + }, + }, + ); + }); + + it('keeps mapping optional for direct identity-header clients', function () { + assert.deepEqual( + importRequestSchema.parse({ filePath: '/tmp/posts.csv', fileName: 'posts.csv' }), + { filePath: '/tmp/posts.csv', fileName: 'posts.csv' }, + ); + }); + + it('rejects empty and prototype CSV headers', function () { + assert.deepEqual(issuesFor({ '': 'title' }), ['Invalid CSV header mapping: ""']); + assert.deepEqual(issuesFor({ constructor: 'title' }), [ + 'Invalid CSV header mapping: "constructor"', + ]); + }); + + it('rejects unknown and duplicate post targets', function () { + assert.deepEqual(issuesFor({ Headline: 'title', Body: 'not_a_field' }), [ + 'Unknown post field mapping: "not_a_field"', + ]); + assert.deepEqual(issuesFor({ Headline: 'title', Duplicate: 'title' }), [ + 'Post field is mapped more than once: "title"', + ]); + }); + + it('requires a title target while allowing multiple ignored columns', function () { + assert.deepEqual(issuesFor({ Body: 'html', Notes: '', Other: '' }), [ + 'Post field mapping must include "title"', + ]); + }); + + it('requires non-empty file paths and names', function () { + const result = importRequestSchema.safeParse({ filePath: '', fileName: '' }); + + if (result.success) { + assert.fail('Expected request validation to fail'); + } + assert.equal(result.error.issues.length, 2); + }); +}); diff --git a/ghost/core/test/unit/server/services/email-analytics/queries.test.ts b/ghost/core/test/unit/server/services/email-analytics/queries.test.ts index fc6797198b38..9042427db06e 100644 --- a/ghost/core/test/unit/server/services/email-analytics/queries.test.ts +++ b/ghost/core/test/unit/server/services/email-analytics/queries.test.ts @@ -214,23 +214,32 @@ describe('Email analytics queries', function () { describe('getLastJobRunTimestamp', function () { it('prefers finished timestamp over started timestamp', async function () { await insertJob({ - started_at: '2026-08-11T10:00:00.000Z', - finished_at: '2026-08-11T11:00:00.000Z', + started_at: new Date('2026-08-11T10:00:00.000Z'), + finished_at: new Date('2026-08-11T11:00:00.000Z'), }); - assert.equal( - await queries.getLastJobRunTimestamp('email-analytics-scheduled'), - '2026-08-11T11:00:00.000Z', - ); + const result = await queries.getLastJobRunTimestamp('email-analytics-scheduled'); + + assert(result instanceof Date); + assert.equal(result.toISOString(), '2026-08-11T11:00:00.000Z'); }); it('falls back to started timestamp', async function () { + await insertJob({ started_at: new Date('2026-08-11T10:00:00.000Z') }); + + const result = await queries.getLastJobRunTimestamp('email-analytics-scheduled'); + + assert(result instanceof Date); + assert.equal(result.toISOString(), '2026-08-11T10:00:00.000Z'); + }); + + it('normalizes string timestamps to Date objects', async function () { await insertJob({ started_at: '2026-08-11T10:00:00.000Z' }); - assert.equal( - await queries.getLastJobRunTimestamp('email-analytics-scheduled'), - '2026-08-11T10:00:00.000Z', - ); + const result = await queries.getLastJobRunTimestamp('email-analytics-scheduled'); + + assert(result instanceof Date); + assert.equal(result.toISOString(), '2026-08-11T10:00:00.000Z'); }); it('returns null when job or timestamps do not exist', async function () { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0c57cd29d6c1..05c5b5cef433 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2371,9 +2371,6 @@ importers: '@tryghost/prometheus-metrics': specifier: 1.0.8 version: 1.0.8(supports-color@10.2.2) - '@tryghost/promise': - specifier: 2.3.2 - version: 2.3.2 '@tryghost/referrer-parser': specifier: 0.1.21 version: 0.1.21 @@ -9463,9 +9460,6 @@ packages: '@tryghost/promise@0.3.20': resolution: {integrity: sha512-afHCBoS72XabqRi7GmHdVAWC/UMsVPGlxnL/epNVp0c/C7tEn6+MgHABXNAW4wViZFhuOda3k7fk3i2K0FAZQg==} - '@tryghost/promise@2.3.2': - resolution: {integrity: sha512-WAwvjo85D5DVHfQ+YhBAb8Vq8h3NS/tfn/gVzrIy/L6TztvSxxPG2t9hU/XVsPbFguEV8lqhInowHdJehW/f0w==} - '@tryghost/promise@2.3.9': resolution: {integrity: sha512-O/d4kTvvHrD/T9Czfhl057uPCJVQulfcQl/XsFHrWOhXewZUc6rsmqTFo6zz9Iv5NRye+SicsUya7BSJFSgKCQ==} @@ -29179,8 +29173,6 @@ snapshots: '@tryghost/promise@0.3.20': {} - '@tryghost/promise@2.3.2': {} - '@tryghost/promise@2.3.9': {} '@tryghost/referrer-parser@0.1.21': {} @@ -30398,9 +30390,9 @@ snapshots: obug: 2.1.3 std-env: 4.1.0 tinyrainbow: 3.1.0 - vitest: 4.1.10(@opentelemetry/api@1.9.1)(@types/node@22.20.1)(@vitest/browser-playwright@4.1.10)(@vitest/coverage-v8@4.1.10)(@vitest/ui@4.1.10)(jsdom@30.0.1(@noble/hashes@1.8.0))(msw@2.14.6(@types/node@22.20.1)(@typescript/typescript6@6.0.2))(vite@8.1.3(@types/node@22.20.1)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.48.0)(tsx@4.23.12)(yaml@2.9.0)) + vitest: 4.1.10(@opentelemetry/api@1.9.1)(@types/node@26.0.0)(@vitest/browser-playwright@4.1.10)(@vitest/coverage-v8@4.1.10)(@vitest/ui@4.1.10)(jsdom@30.0.1(@noble/hashes@1.8.0))(msw@2.14.6(@types/node@26.0.0)(@typescript/typescript6@6.0.2))(vite@8.1.3(@types/node@26.0.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.48.0)(tsx@4.23.12)(yaml@2.9.0)) optionalDependencies: - '@vitest/browser': 4.1.10(msw@2.14.6(@types/node@22.20.1)(@typescript/typescript6@6.0.2))(vite@8.1.3(@types/node@22.20.1)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.48.0)(tsx@4.23.12)(yaml@2.9.0))(vitest@4.1.10) + '@vitest/browser': 4.1.10(msw@2.14.6(@types/node@26.0.0)(@typescript/typescript6@6.0.2))(vite@8.1.3(@types/node@26.0.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.48.0)(tsx@4.23.12)(yaml@2.9.0))(vitest@4.1.10) '@vitest/expect@3.2.4': dependencies: @@ -30481,7 +30473,7 @@ snapshots: sirv: 3.0.2 tinyglobby: 0.2.17 tinyrainbow: 3.1.0 - vitest: 4.1.10(@opentelemetry/api@1.9.1)(@types/node@22.20.1)(@vitest/browser-playwright@4.1.10)(@vitest/coverage-v8@4.1.10)(@vitest/ui@4.1.10)(jsdom@30.0.1(@noble/hashes@1.8.0))(msw@2.14.6(@types/node@22.20.1)(@typescript/typescript6@6.0.2))(vite@8.1.3(@types/node@22.20.1)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.48.0)(tsx@4.23.12)(yaml@2.9.0)) + vitest: 4.1.10(@opentelemetry/api@1.9.1)(@types/node@26.0.0)(@vitest/browser-playwright@4.1.10)(@vitest/coverage-v8@4.1.10)(@vitest/ui@4.1.10)(jsdom@30.0.1(@noble/hashes@1.8.0))(msw@2.14.6(@types/node@26.0.0)(@typescript/typescript6@6.0.2))(vite@8.1.3(@types/node@26.0.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.48.0)(tsx@4.23.12)(yaml@2.9.0)) '@vitest/utils@3.2.4': dependencies: diff --git a/scripts/lib/utils.js b/scripts/lib/utils.js index 7ba6078b0f5d..1e7bb8f4fdc7 100644 --- a/scripts/lib/utils.js +++ b/scripts/lib/utils.js @@ -15,10 +15,10 @@ export const readJsonSync = (filePath) => JSON.parse(readFileSync(filePath, 'utf * @param {object} json */ export const writeJson = async (filePath, json) => - await writeFile(filePath, JSON.stringify(json, null, 4) + '\n', 'utf8'); + await writeFile(filePath, JSON.stringify(json, null, 2) + '\n', 'utf8'); /** * @param {string} filePath * @param {object} json */ export const writeJsonSync = (filePath, json) => - writeFileSync(filePath, JSON.stringify(json, null, 4) + '\n', 'utf8'); + writeFileSync(filePath, JSON.stringify(json, null, 2) + '\n', 'utf8'); diff --git a/scripts/release.js b/scripts/release.js index c5d0d166a088..c167ff77b509 100644 --- a/scripts/release.js +++ b/scripts/release.js @@ -50,6 +50,26 @@ function run(cmd, opts = {}) { return result.trim(); } +// Single-quote a path for the shell. +function quoteArg(path) { + return `'${path.replace(/'/g, `'\\''`)}'`; +} + +// Parse `git status --porcelain=v1 -z`. R/C entries append a source path field +// (destination first), so skip it. +function parsePorcelainPaths(porcelain) { + const fields = porcelain.split('\0').filter(Boolean); + const entries = []; + for (let i = 0; i < fields.length; i++) { + const status = fields[i].slice(0, 2); + entries.push({ path: fields[i].slice(3), deleted: status.includes('D') }); + if (status[0] === 'R' || status[0] === 'C') { + i += 1; + } + } + return entries; +} + function readPkgVersion(pkgPath) { return readJsonSync(pkgPath).version; } @@ -297,19 +317,35 @@ async function runPackagesOnlyRelease(opts) { log(`Branch: ${opts.branch}`); log(`Dry run: ${opts.dryRun}`); + // Release runs on a fresh CI checkout. Bail on a dirty tree so unrelated + // changes can't ride along in the format + commit below. + if (run('git status --porcelain=v1 -z')) { + throw new Error('Working tree is not clean; refusing to release'); + } + logStep('Applying changeset versions to publishable packages'); applyChangesetVersions(); // version -r writes package.json bumps, rewrites workspace ranges, removes // consumed changesets, and may touch the lockfile. Nothing staged means // there were no pending changesets to release. - const changes = run('git status --porcelain'); + const changes = run('git status --porcelain=v1 -z'); if (!changes) { log('No pending package changes to release'); console.log('\n✓ Nothing to publish'); return; } + // Format only touched files so oxfmt doesn't reformat the whole repo. + const filesToFormat = parsePorcelainPaths(changes) + .filter((f) => !f.deleted) + .map((f) => f.path); + + logStep('Formatting changed files'); + if (filesToFormat.length) { + run(`pnpm format ${filesToFormat.map(quoteArg).join(' ')}`); + } + logStep('Committing package versions'); run('git add -A'); run('git commit -m "Versioned pending package changesets"');