{{@site.description}}
{{/if}} {{#if @site.allow_self_signup}} - + {{#if @member}} + {{> private-access-form}} + {{else}} + + {{/if}} {{/if}} {{#unless @site.allow_self_signup}} - + {{> private-access-form}} {{/unless}} {{#if @site.allow_self_signup}} + {{#unless @member}} + {{/unless}} {{/if}}
diff --git a/ghost/core/core/frontend/helpers/cancel_link.js b/ghost/core/core/frontend/helpers/cancel_link.js
index 94492f5dc51..a048db4d792 100644
--- a/ghost/core/core/frontend/helpers/cancel_link.js
+++ b/ghost/core/core/frontend/helpers/cancel_link.js
@@ -13,7 +13,7 @@ const tpl = require('@tryghost/tpl');
const messages = {
invalidData:
- 'The {{cancel_link}} helper was used outside of a subscription context. See https://ghost.org/docs/themes/members/#cancel-links.',
+ 'The {{cancel_link}} helper was used outside of a subscription context. See https://docs.ghost.org/themes/members/#cancel-links.',
};
// eslint-disable-next-line camelcase
@@ -45,7 +45,7 @@ module.exports = function cancelLabsWrapper() {
flagKey: 'members',
flagName: 'Members',
helperName: 'cancel_link',
- helpUrl: 'https://ghost.org/docs/themes/members/',
+ helpUrl: 'https://docs.ghost.org/themes/members/',
},
() => {
return cancel_link.apply(self, args); // eslint-disable-line camelcase
diff --git a/ghost/core/core/frontend/helpers/pagination.js b/ghost/core/core/frontend/helpers/pagination.js
index 189578a4b0a..478b581d3e8 100644
--- a/ghost/core/core/frontend/helpers/pagination.js
+++ b/ghost/core/core/frontend/helpers/pagination.js
@@ -9,7 +9,7 @@ const _ = require('lodash');
const messages = {
invalidData:
- 'The {{pagination}} helper was used outside of a paginated context. See https://ghost.org/docs/themes/helpers/pagination/.',
+ 'The {{pagination}} helper was used outside of a paginated context. See https://docs.ghost.org/themes/helpers/pagination/.',
valuesMustBeDefined: 'All values must be defined for page, pages, limit and total',
nextPrevValuesMustBeNumeric: 'Invalid value, Next/Prev must be a number',
valuesMustBeNumeric: 'Invalid value, check page, pages, limit and total are numbers',
@@ -26,7 +26,7 @@ module.exports = function pagination(options) {
throw new errors.IncorrectUsageError({
level: 'normal',
message: tpl(messages.invalidData),
- help: 'https://ghost.org/docs/themes/helpers/pagination/',
+ help: 'https://docs.ghost.org/themes/helpers/pagination/',
});
}
diff --git a/ghost/core/core/frontend/helpers/social_accounts.js b/ghost/core/core/frontend/helpers/social_accounts.js
index db4c4ca3578..d3bbbca9aae 100644
--- a/ghost/core/core/frontend/helpers/social_accounts.js
+++ b/ghost/core/core/frontend/helpers/social_accounts.js
@@ -54,7 +54,7 @@ module.exports = function social_accounts(source, options) {
throw new errors.IncorrectUsageError({
level: 'normal',
message: tpl(messages.sourceRequired),
- help: 'https://ghost.org/docs/themes/helpers/social-accounts/',
+ help: 'https://docs.ghost.org/themes/helpers/social-accounts/',
});
}
@@ -67,7 +67,7 @@ module.exports = function social_accounts(source, options) {
throw new errors.IncorrectUsageError({
level: 'normal',
message: tpl(messages.blockRequired),
- help: 'https://ghost.org/docs/themes/helpers/social-accounts/',
+ help: 'https://docs.ghost.org/themes/helpers/social-accounts/',
});
}
diff --git a/ghost/core/core/frontend/public/ghost.css b/ghost/core/core/frontend/public/ghost.css
index 844389b702a..8be5b44ed1a 100644
--- a/ghost/core/core/frontend/public/ghost.css
+++ b/ghost/core/core/frontend/public/ghost.css
@@ -740,7 +740,7 @@ h2 {
}
.gh-private-signup-form,
-.gh-flow-content.private > .gh-private-access-form {
+.gh-flow-content.private .gh-private-access-form {
width: min(100%, 550px);
margin: 40px auto 0;
}
diff --git a/ghost/core/core/frontend/services/sitemap/user-map-generator.js b/ghost/core/core/frontend/services/sitemap/user-map-generator.js
index db3a2f8a618..6ecba48e90a 100644
--- a/ghost/core/core/frontend/services/sitemap/user-map-generator.js
+++ b/ghost/core/core/frontend/services/sitemap/user-map-generator.js
@@ -11,9 +11,16 @@ class UserMapGenerator extends BaseMapGenerator {
}
validateImageUrl(imageUrl) {
+ /** @type {Parameters[1]} */
+ const isUrlOptions = {
+ protocols: ['http', 'https'],
+ require_protocol: true,
+ };
return (
- imageUrl &&
- validator.isURL(imageUrl, { protocols: ['http', 'https'], require_protocol: true })
+ typeof imageUrl === 'string' &&
+ // `validator.isURL` doesn't let us express "any TLD or localhost", so we do two checks.
+ (validator.isURL(imageUrl, isUrlOptions) ||
+ validator.isURL(imageUrl, { ...isUrlOptions, host_whitelist: ['localhost'] }))
);
}
}
diff --git a/ghost/core/core/server/api/endpoints/mail.js b/ghost/core/core/server/api/endpoints/mail.js
index d104f33aed9..7fa13818681 100644
--- a/ghost/core/core/server/api/endpoints/mail.js
+++ b/ghost/core/core/server/api/endpoints/mail.js
@@ -25,7 +25,7 @@ _private.sendMail = (object) => {
message: [
tpl(messages.unableToSendEmail),
tpl(messages.seeLinkForInstructions, {
- link: 'https://ghost.org/docs/concepts/config/#mail',
+ link: 'https://docs.ghost.org/concepts/config/#mail',
}),
].join(' '),
},
diff --git a/ghost/core/core/server/api/endpoints/posts.js b/ghost/core/core/server/api/endpoints/posts.js
index 4af5409fe3b..add78359041 100644
--- a/ghost/core/core/server/api/endpoints/posts.js
+++ b/ghost/core/core/server/api/endpoints/posts.js
@@ -136,6 +136,7 @@ const controller = {
filePath: frame.file.path,
fileName: frame.file.name,
mapping: frame.data.mapping,
+ requestUserEmail: frame.user ? frame.user.get('email') : null,
});
return { meta: { import_id: importId, total } };
},
diff --git a/ghost/core/core/server/api/endpoints/utils/validators/input/pages.js b/ghost/core/core/server/api/endpoints/utils/validators/input/pages.js
index 958ee492615..8deda45ac1c 100644
--- a/ghost/core/core/server/api/endpoints/utils/validators/input/pages.js
+++ b/ghost/core/core/server/api/endpoints/utils/validators/input/pages.js
@@ -8,7 +8,7 @@ const messages = {
onlySingleContentSource: 'Pages can have either a mobiledoc or a lexical property, never both.',
onlySingleContentSourceContext:
'Both the mobiledoc and lexical properties are set, one must be null',
- onlySingleContentSourceHelp: 'https://ghost.org/docs/admin-api/#the-post-object',
+ onlySingleContentSourceHelp: 'https://docs.ghost.org/admin-api/#the-post-object',
};
const validateVisibility = async function (frame) {
diff --git a/ghost/core/core/server/api/endpoints/utils/validators/input/posts.js b/ghost/core/core/server/api/endpoints/utils/validators/input/posts.js
index 43324e5a8dd..2a23081e0b7 100644
--- a/ghost/core/core/server/api/endpoints/utils/validators/input/posts.js
+++ b/ghost/core/core/server/api/endpoints/utils/validators/input/posts.js
@@ -8,7 +8,7 @@ const messages = {
onlySingleContentSource: 'Posts can have either a mobiledoc or a lexical property, never both.',
onlySingleContentSourceContext:
'Both the mobiledoc and lexical properties are set, one must be null',
- onlySingleContentSourceHelp: 'https://ghost.org/docs/admin-api/#the-post-object',
+ onlySingleContentSourceHelp: 'https://docs.ghost.org/admin-api/#the-post-object',
};
const validateVisibility = async function (frame) {
diff --git a/ghost/core/core/server/data/importer/importers/data/data-importer.js b/ghost/core/core/server/data/importer/importers/data/data-importer.js
index b29df8c2a89..e74a4e16dff 100644
--- a/ghost/core/core/server/data/importer/importers/data/data-importer.js
+++ b/ghost/core/core/server/data/importer/importers/data/data-importer.js
@@ -99,7 +99,7 @@ DataImporter = {
return Promise.reject(
new IncorrectUsageError({
message: 'Wrong importer structure. `meta` is missing.',
- help: 'https://ghost.org/docs/migration/custom/',
+ help: 'https://docs.ghost.org/migration/custom/',
}),
);
}
@@ -108,7 +108,7 @@ DataImporter = {
return Promise.reject(
new IncorrectUsageError({
message: 'Wrong importer structure. `meta.version` is missing.',
- help: 'https://ghost.org/docs/migration/custom/',
+ help: 'https://docs.ghost.org/migration/custom/',
}),
);
}
@@ -119,7 +119,7 @@ DataImporter = {
return Promise.reject(
new IncorrectUsageError({
message: 'Detected unsupported file structure.',
- help: 'Please install Ghost 1.0, import the file and then update your blog to the latest Ghost version.\nVisit https://ghost.org/docs/update/ or ask for help in our https://forum.ghost.org.',
+ help: 'Please install Ghost 1.0, import the file and then update your blog to the latest Ghost version.\nVisit https://docs.ghost.org/update/ or ask for help in our https://forum.ghost.org.',
}),
);
}
diff --git a/ghost/core/core/server/data/migrations/utils/migrations.js b/ghost/core/core/server/data/migrations/utils/migrations.js
index 2d29b3fc851..fc4c5414a0c 100644
--- a/ghost/core/core/server/data/migrations/utils/migrations.js
+++ b/ghost/core/core/server/data/migrations/utils/migrations.js
@@ -117,7 +117,7 @@ function createFinalMigration(major) {
async function up() {
throw new errors.InternalServerError({
message: `Unable to run migrations`,
- context: `You must be on the latest v${major}.x to update across major versions - https://ghost.org/docs/update/`,
+ context: `You must be on the latest v${major}.x to update across major versions - https://docs.ghost.org/update/`,
help: `Run 'ghost update v${major}' to get the latest v${major}.x version, then run 'ghost update' to get to the latest.`,
});
},
diff --git a/ghost/core/core/server/lib/package-json/parse.js b/ghost/core/core/server/lib/package-json/parse.js
index 011616da5da..9be554df3b0 100644
--- a/ghost/core/core/server/lib/package-json/parse.js
+++ b/ghost/core/core/server/lib/package-json/parse.js
@@ -37,7 +37,7 @@ async function parse(path) {
message: tpl(messages.themeFileIsMalformed),
context: path,
err: parseError,
- help: tpl(messages.willBeRequired, { url: 'https://ghost.org/docs/themes/' }),
+ help: tpl(messages.willBeRequired, { url: 'https://docs.ghost.org/themes/' }),
});
}
@@ -47,7 +47,7 @@ async function parse(path) {
throw new errors.IncorrectUsageError({
message: tpl(messages.nameOrVersionMissing),
context: path,
- help: tpl(messages.willBeRequired, { url: 'https://ghost.org/docs/themes/' }),
+ help: tpl(messages.willBeRequired, { url: 'https://docs.ghost.org/themes/' }),
});
}
diff --git a/ghost/core/core/server/models/post.js b/ghost/core/core/server/models/post.js
index 2bd66c350c0..1b21a5b8407 100644
--- a/ghost/core/core/server/models/post.js
+++ b/ghost/core/core/server/models/post.js
@@ -29,9 +29,9 @@ const messages = {
notEnoughPermission: 'You do not have permission to perform this action',
invalidNewsletter: "The newsletter parameter doesn't match any active newsletter.",
invalidMobiledocStructure: 'Invalid mobiledoc structure.',
- invalidMobiledocStructureHelp: 'https://ghost.org/docs/publishing/',
+ invalidMobiledocStructureHelp: 'https://docs.ghost.org/publishing/',
invalidLexicalStructure: 'Invalid lexical structure.',
- invalidLexicalStructureHelp: 'https://ghost.org/docs/publishing/',
+ invalidLexicalStructureHelp: 'https://docs.ghost.org/publishing/',
emailOnlyWithoutNewsletter: 'Scheduling an email requires a newsletter reference.',
};
diff --git a/ghost/core/core/server/models/relations/authors.js b/ghost/core/core/server/models/relations/authors.js
index 37e999566c8..671cf7512d6 100644
--- a/ghost/core/core/server/models/relations/authors.js
+++ b/ghost/core/core/server/models/relations/authors.js
@@ -149,7 +149,6 @@ module.exports.extendModel = function extendModel(Post, Posts, ghostBookshelf) {
// CASE: `posts.authors` was not requested, but fetched in specific cases (see top)
if (
- !this._originalOptions ||
!this._originalOptions.withRelated ||
this._originalOptions.withRelated.indexOf('authors') === -1
) {
@@ -157,10 +156,7 @@ module.exports.extendModel = function extendModel(Post, Posts, ghostBookshelf) {
}
// If the current column settings allow it...
- if (
- !options.columns ||
- (options.columns && options.columns.indexOf('primary_author') > -1)
- ) {
+ if (!options.columns || options.columns.indexOf('primary_author') > -1) {
// ... attach a computed property of primary_author which is the first author
if (attrs.authors && attrs.authors.length) {
attrs.primary_author = attrs.authors[0];
@@ -398,20 +394,10 @@ module.exports.extendModel = function extendModel(Post, Posts, ghostBookshelf) {
}
function isOwner() {
- let isCorrectOwner = true;
-
if (!unsafeAttrs.authors) {
return false;
}
-
- if (unsafeAttrs.authors) {
- isCorrectOwner =
- isCorrectOwner &&
- unsafeAttrs.authors.length &&
- unsafeAttrs.authors[0].id === context.user;
- }
-
- return isCorrectOwner;
+ return unsafeAttrs.authors.length && unsafeAttrs.authors[0].id === context.user;
}
function isPrimaryAuthor() {
diff --git a/ghost/core/core/server/services/content-import/csv/formula.ts b/ghost/core/core/server/services/content-import/csv/formula.ts
new file mode 100644
index 00000000000..55cf006bc10
--- /dev/null
+++ b/ghost/core/core/server/services/content-import/csv/formula.ts
@@ -0,0 +1,10 @@
+const FORMULA_TRIGGERS = ['=', '+', '-', '@', '\t', '\r'];
+
+// Report files prefix formula-shaped cells with an apostrophe. Remove exactly
+// that guard during re-import so repeated fixes do not change publisher data.
+export function stripFormulaGuard(cell: string): string {
+ if (cell.charAt(0) === "'" && FORMULA_TRIGGERS.includes(cell.charAt(1))) {
+ return cell.slice(1);
+ }
+ return cell;
+}
diff --git a/ghost/core/core/server/services/content-import/csv/index.ts b/ghost/core/core/server/services/content-import/csv/index.ts
index edebe083a66..703851cdd1f 100644
--- a/ghost/core/core/server/services/content-import/csv/index.ts
+++ b/ghost/core/core/server/services/content-import/csv/index.ts
@@ -1 +1,9 @@
-export { default as parse, type Row } from './parse';
+export {
+ default as parse,
+ parseWithSource,
+ type ParsedCSV,
+ type ParsedRow,
+ type Row,
+} from './parse';
+export { default as serialize } from './serialize';
+export { stripFormulaGuard } from './formula';
diff --git a/ghost/core/core/server/services/content-import/csv/parse.ts b/ghost/core/core/server/services/content-import/csv/parse.ts
index 727d3029b8f..a8b725c1fad 100644
--- a/ghost/core/core/server/services/content-import/csv/parse.ts
+++ b/ghost/core/core/server/services/content-import/csv/parse.ts
@@ -1,11 +1,23 @@
import papaparse from 'papaparse';
import fs from 'fs-extra';
+import { stripFormulaGuard } from './formula';
const errors = require('@tryghost/errors');
// A parsed CSV row: raw string cells, keyed by (renamed) column.
export type Row = Record;
+export interface ParsedRow {
+ data: Row;
+ source: Row;
+ line: number;
+}
+
+export interface ParsedCSV {
+ columns: string[];
+ rows: ParsedRow[];
+}
+
// A column named after an Object.prototype member is unsafe as a key.
function isSafeColumnName(name: string): boolean {
return !(name in Object.prototype);
@@ -18,19 +30,38 @@ function isFatal(error: papaparse.ParseError): boolean {
return error.type === 'Quotes';
}
+function isEmptyRow(row: Record): boolean {
+ return !Object.values(row).some((value) => {
+ if (Array.isArray(value)) {
+ return value.some((cell) => typeof cell === 'string' && cell.trim().length > 0);
+ }
+ return value.trim().length > 0;
+ });
+}
+
// headerMapping renames headers to the columns they emit under; unmapped columns
// carry through, and one mapped to an empty target is dropped.
export default async function parse(
path: string,
headerMapping?: Record,
): Promise {
+ const parsed = await parseWithSource(path, headerMapping);
+ return parsed.rows.map(({ data }) => data);
+}
+
+export async function parseWithSource(
+ path: string,
+ headerMapping?: Record,
+): Promise {
// Buffered rather than streamed: files are bounded by the interim row cap, and
// papaparse's stream mode drops results.errors, which is what catches a
// malformed quoted field before it imports as garbage.
const content = (await fs.readFile(path, 'utf8')).replace(/^\ufeff/, '');
- const parsed = papaparse.parse>(content, {
+ const parsed = papaparse.parse>(content, {
header: true,
- skipEmptyLines: true,
+ // Empty records stay in the parser output long enough to contribute to the
+ // publisher-visible spreadsheet line number, then are dropped below.
+ skipEmptyLines: false,
});
const fatal = parsed.errors.find(isFatal);
@@ -38,10 +69,15 @@ export default async function parse(
throw new errors.ValidationError({ message: `${fatal.code}: ${fatal.message}` });
}
- const rows: Row[] = [];
+ const rows: ParsedRow[] = [];
+
+ for (const [index, parsedRow] of parsed.data.entries()) {
+ if (isEmptyRow(parsedRow)) {
+ continue;
+ }
- for (const parsedRow of parsed.data) {
const row: Row = {};
+ const source: Row = Object.create(null) as Row;
for (const [header, value] of Object.entries(parsedRow)) {
// non-string values are papaparse's __parsed_extra overflow from ragged rows
@@ -49,14 +85,16 @@ export default async function parse(
continue;
}
+ source[header] = value;
+
// hasOwn: a prototype-named header must not match an inherited method on the mapping
if (headerMapping && Object.hasOwn(headerMapping, header)) {
if (!headerMapping[header]) {
continue;
}
- row[headerMapping[header]] = value;
+ row[headerMapping[header]] = stripFormulaGuard(value);
} else if (isSafeColumnName(header)) {
- row[header] = value;
+ row[header] = stripFormulaGuard(value);
}
}
@@ -64,8 +102,8 @@ export default async function parse(
continue;
}
- rows.push(row);
+ rows.push({ data: row, source, line: index + 2 });
}
- return rows;
+ return { columns: parsed.meta.fields ?? [], rows };
}
diff --git a/ghost/core/core/server/services/content-import/csv/serialize.ts b/ghost/core/core/server/services/content-import/csv/serialize.ts
new file mode 100644
index 00000000000..decba6d53a4
--- /dev/null
+++ b/ghost/core/core/server/services/content-import/csv/serialize.ts
@@ -0,0 +1,16 @@
+import papaparse from 'papaparse';
+
+interface SerializeOptions {
+ columns?: string[];
+ header?: boolean;
+}
+
+// Import reports are opened in spreadsheets, so formula-shaped cells must never
+// be emitted verbatim. Papa Parse prefixes them with an apostrophe when this
+// option is enabled.
+export default function serialize(
+ rows: Array>,
+ { columns, header = true }: SerializeOptions = {},
+): string {
+ return papaparse.unparse(rows, { columns, header, escapeFormulae: true });
+}
diff --git a/ghost/core/core/server/services/content-import/import/completion-email.ts b/ghost/core/core/server/services/content-import/import/completion-email.ts
new file mode 100644
index 00000000000..95de987e550
--- /dev/null
+++ b/ghost/core/core/server/services/content-import/import/completion-email.ts
@@ -0,0 +1,199 @@
+import type { ImportRun, RowStatus } from './store';
+import buildImportReport from './report';
+import buildErrorsFile from './errors-file';
+
+const { slugify } = require('@tryghost/string');
+
+const IMPORTED_POST_PREVIEW_LIMIT = 10;
+
+export interface CompletionEmailPayload {
+ to: string;
+ subject: string;
+ html: string;
+ forceTextContent: boolean;
+ attachments: Array<{
+ filename: string;
+ content: string;
+ contentType: string;
+ contentDisposition: string;
+ }>;
+}
+
+type OutcomeCounts = Record & { warningRows: number };
+
+function escapeHTML(value: string): string {
+ return value
+ .replace(/&/g, '&')
+ .replace(//g, '>')
+ .replace(/"/g, '"')
+ .replace(/'/g, ''');
+}
+
+function countsFor(run: ImportRun): OutcomeCounts {
+ const counts: OutcomeCounts = {
+ created: 0,
+ updated: 0,
+ skipped: 0,
+ failed: 0,
+ warningRows: 0,
+ };
+
+ for (const row of run.rows) {
+ counts[row.status] += 1;
+ if (row.warnings?.length) {
+ counts.warningRows += 1;
+ }
+ }
+
+ return counts;
+}
+
+function headingFor(run: ImportRun, counts: OutcomeCounts): string {
+ if (run.status === 'failed') {
+ return 'Your content import could not be completed';
+ }
+ if (counts.failed > 0 && counts.created + counts.updated === 0) {
+ return 'Your content import was unsuccessful';
+ }
+ return 'Your content import is complete';
+}
+
+function formatNumber(value: number): string {
+ return value.toLocaleString();
+}
+
+function summaryItems(counts: OutcomeCounts): string {
+ const items = [
+ ['Created', counts.created],
+ ['Updated', counts.updated],
+ ['Skipped', counts.skipped],
+ ['Failed', counts.failed],
+ ];
+
+ return items
+ .map(
+ ([label, count]) =>
+ `- ${label}: ${formatNumber(count as number)}
`,
+ )
+ .join('');
+}
+
+function importTagSlug(runId: string): string {
+ // Internal tag slugs replace the leading # with "hash-" in the model layer.
+ return slugify(`#Import Run ${runId}`.replace(/^#/, 'hash-'));
+}
+
+function importedPostLinks(run: ImportRun, adminUrl: string): string {
+ const importedRows = run.rows.filter(
+ (row) => row.status === 'created' || row.status === 'updated',
+ );
+ if (!importedRows.length) {
+ return '';
+ }
+
+ const rows = importedRows.filter((row) => row.url);
+ const previewRows = rows.slice(0, IMPORTED_POST_PREVIEW_LIMIT);
+ const hasPages = importedRows.some((row) => row.postType === 'page');
+ const hasPosts = importedRows.some((row) => row.postType !== 'page');
+
+ const links = previewRows
+ .map((row) => {
+ const title = escapeHTML(row.title || `Row ${row.line}`);
+ return `- ${title}
`;
+ })
+ .join('');
+ const remaining = importedRows.length - previewRows.length;
+ const remainingCopy = remaining
+ ? `${heading}
+
+
+ ): string {
+ let candidate = preferred;
+ let suffix = 2;
+ while (used.has(candidate)) {
+ candidate = `${preferred}_${suffix}`;
+ suffix += 1;
+ }
+ used.add(candidate);
+ return candidate;
+}
+
+export default function buildErrorsFile(run: ImportRun): string | undefined {
+ const rows = run.rows.filter(isActionable);
+ if (!rows.length) {
+ return undefined;
+ }
+
+ const usedColumns = new Set(run.sourceColumns);
+ const [outcomeColumn, reasonColumn, mediaFailuresColumn] = ANNOTATION_NAMES.map((name) =>
+ uniqueColumnName(name, usedColumns),
+ );
+ const columns = [outcomeColumn, ...run.sourceColumns, reasonColumn, mediaFailuresColumn];
+
+ return serialize(
+ rows.map((row) => {
+ const sourceCells = Object.fromEntries(
+ run.sourceColumns.map((column) => [column, row.source?.[column] ?? '']),
+ );
+ return {
+ ...sourceCells,
+ [outcomeColumn]: row.status,
+ [reasonColumn]: row.reason ?? '',
+ [mediaFailuresColumn]: row.mediaFailures?.length ? JSON.stringify(row.mediaFailures) : '',
+ };
+ }),
+ { columns },
+ );
+}
diff --git a/ghost/core/core/server/services/content-import/import/importer.ts b/ghost/core/core/server/services/content-import/import/importer.ts
index 12c732d9b78..c36b6fe3f33 100644
--- a/ghost/core/core/server/services/content-import/import/importer.ts
+++ b/ghost/core/core/server/services/content-import/import/importer.ts
@@ -9,8 +9,9 @@ import buildPostData, {
import type { PostImportRow } from './row';
import type { PostsRepository, WrittenPost } from './post-repository';
import type { ImportRequest } from './schema';
-import type { Clock, ImportRunStore, RowOutcome } from './store';
+import type { Clock, ImportRun, ImportRunStore, RowOutcome } from './store';
import type { PreparedImportSource } from './source';
+import type { PreparedPostRow, PreparedPostRows } from './reader';
import { MediaInliningFailure, type PostMediaInlining } from './media';
export type { ImportRequest } from './schema';
@@ -32,7 +33,15 @@ export interface ImportAccepted {
// Must not throw: it is called from catch blocks that exist to stop an error escaping.
export type FailureReporter = (error: unknown) => void;
-type ReadRows = (path: string, mapping?: Record) => Promise;
+export interface EmailNotifications {
+ send(run: ImportRun, recipient: string): Promise;
+ getDefaultRecipient(): Promise;
+}
+
+type ReadRows = (
+ path: string,
+ mapping?: Record,
+) => Promise;
type PrepareSource = (request: ImportRequest) => Promise;
const messages = {
@@ -62,6 +71,7 @@ interface ImporterDeps {
getMarkdownToHtml: () => MarkdownToHtml;
getCleanHTML: () => CleanHTML;
createMediaInliner: () => PostMediaInlining;
+ email: EmailNotifications;
addJob: (job: { job: () => Promise; offloaded: boolean; name: string }) => void;
report: FailureReporter;
store: ImportRunStore;
@@ -86,6 +96,7 @@ class ContentCSVImporter {
private _getMarkdownToHtml: () => MarkdownToHtml;
private _getCleanHTML: () => CleanHTML;
private _createMediaInliner: () => PostMediaInlining;
+ private _email: EmailNotifications;
private _addJob: ImporterDeps['addJob'];
private _report: FailureReporter;
private _store: ImportRunStore;
@@ -102,6 +113,7 @@ class ContentCSVImporter {
getMarkdownToHtml,
getCleanHTML,
createMediaInliner,
+ email,
addJob,
report,
store,
@@ -117,6 +129,7 @@ class ContentCSVImporter {
this._getMarkdownToHtml = getMarkdownToHtml;
this._getCleanHTML = getCleanHTML;
this._createMediaInliner = createMediaInliner;
+ this._email = email;
this._addJob = addJob;
this._report = report;
this._store = store;
@@ -127,10 +140,14 @@ class ContentCSVImporter {
}
async importCSV(request: ImportRequest): Promise {
+ const emailRecipient = request.requestUserEmail ?? (await this._email.getDefaultRecipient());
const source = await this._prepareSource(request);
- let rows: PostImportRow[];
+ let preparedRows: PreparedPostRows;
try {
- rows = await this._readRows(source.filePath, request.mapping);
+ const result = await this._readRows(source.filePath, request.mapping);
+ preparedRows = Array.isArray(result)
+ ? { columns: [], rows: result.map((data, index) => ({ data, line: index + 2 })) }
+ : result;
} catch (error) {
await this.cleanupSource(source.cleanup);
throw new errors.ValidationError({
@@ -141,7 +158,7 @@ class ContentCSVImporter {
// Temporary while import state is held in memory: the durable job
// system milestone removes the cap.
- if (rows.length > MAX_POSTS) {
+ if (preparedRows.rows.length > MAX_POSTS) {
await this.cleanupSource(source.cleanup);
throw new errors.ValidationError({
message: tpl(messages.tooManyPosts, { max: MAX_POSTS }),
@@ -150,22 +167,24 @@ class ContentCSVImporter {
const runId = this._newRunId();
const importTagNames = buildImportTagNames(runId, this._getTimezone(), this._now());
- this._store.create(runId, rows.length);
+ this._store.create(runId, preparedRows.rows.length, preparedRows.columns);
logLifecycle('queued');
try {
this._addJob({
- job: () => this.runImportJob(runId, importTagNames, rows, source),
+ job: () =>
+ this.runImportJob(runId, importTagNames, preparedRows.rows, source, emailRecipient),
offloaded: false,
name: 'content-import',
});
} catch (error) {
this._store.fail(runId, messageOf(error));
await this.cleanupSource(source.cleanup);
+ this._store.release(runId);
throw error;
}
- return { importId: runId, total: rows.length };
+ return { importId: runId, total: preparedRows.rows.length };
}
// Must resolve in every case: the job manager reads a rejected inline job as a
@@ -173,8 +192,9 @@ class ContentCSVImporter {
private async runImportJob(
runId: string,
importTagNames: string[],
- rows: PostImportRow[],
+ rows: PreparedPostRow[],
source: PreparedImportSource,
+ emailRecipient: string,
): Promise {
const startedAt = Date.now();
logLifecycle('started');
@@ -185,7 +205,7 @@ class ContentCSVImporter {
try {
if (source.assets) {
await source.assets.store();
- source.assets.rewriteRows(rows);
+ source.assets.rewriteRows(rows.map(({ data }) => data));
}
const htmlToLexical = this._getHtmlToLexical();
@@ -196,18 +216,18 @@ class ContentCSVImporter {
let failedRows = 0;
let firstRowFailure: unknown;
- for (const [index, row] of rows.entries()) {
- const line = index + 2;
+ for (const preparedRow of rows) {
+ const { data: row, source: sourceCells, line } = preparedRow;
let data: PostData;
try {
data = buildPostData(row, htmlToLexical, importTagNames, markdownToHtml, cleanHTML);
} catch (error) {
if (error instanceof RowSkipped) {
- this._store.record(runId, {
+ this.recordOutcome(runId, sourceCells, {
line,
title: row.title || null,
- status: 'skipped',
+ status: 'failed',
reason: messageOf(error),
});
continue;
@@ -226,7 +246,7 @@ class ContentCSVImporter {
firstRowFailure = error;
}
failedRows += 1;
- this._store.record(runId, {
+ this.recordOutcome(runId, sourceCells, {
line,
title: row.title,
status: 'failed',
@@ -257,6 +277,7 @@ class ContentCSVImporter {
},
{
sourceUpdatedAt: row.updated_at,
+ runTagName: importTagNames[1],
authorNames: row.authors,
authorEmails: row.author_emails,
tagNames: row.tags,
@@ -264,11 +285,12 @@ class ContentCSVImporter {
);
if (result.status === 'skipped') {
- this._store.record(runId, {
+ this.recordOutcome(runId, sourceCells, {
line,
title: row.title,
status: 'skipped',
reason: result.reason,
+ duplicate: result.duplicate,
});
continue;
}
@@ -282,7 +304,7 @@ class ContentCSVImporter {
firstRowFailure = error;
}
failedRows += 1;
- this._store.record(runId, {
+ this.recordOutcome(runId, sourceCells, {
line,
title: row.title,
status: 'failed',
@@ -296,6 +318,7 @@ class ContentCSVImporter {
title: row.title,
status: writeStatus,
postId: post.id,
+ postType: data.type,
...(warnings.length > 0 ? { warnings } : {}),
};
try {
@@ -306,7 +329,7 @@ class ContentCSVImporter {
}
urlFailureCount += 1;
}
- this._store.record(runId, outcome);
+ this.recordOutcome(runId, sourceCells, outcome);
}
if (failedRows > 0 && successfulWrites === 0) {
@@ -330,11 +353,24 @@ class ContentCSVImporter {
this._store.fail(runId, messageOf(error));
} finally {
await this.cleanupSource(source.cleanup);
+ const run = this._store.get(runId);
+ if (run) {
+ await this.settle(() => this._email.send(run, emailRecipient));
+ }
+ this._store.release(runId);
const outcome = failed ? 'failed after' : 'completed in';
logLifecycle(`${outcome} ${Date.now() - startedAt}ms`);
}
}
+ private recordOutcome(
+ runId: string,
+ source: Record | undefined,
+ outcome: RowOutcome,
+ ): void {
+ this._store.record(runId, source ? { ...outcome, source } : outcome);
+ }
+
private reportUrlFailures(count: number, firstFailure: unknown): void {
if (count > 0) {
this._report(
@@ -356,6 +392,14 @@ class ContentCSVImporter {
this._report(error);
}
}
+
+ private async settle(operation: () => Promise): Promise {
+ try {
+ await operation();
+ } catch (error) {
+ this._report(error);
+ }
+ }
}
function messageOf(error: unknown): string {
diff --git a/ghost/core/core/server/services/content-import/import/post-link.ts b/ghost/core/core/server/services/content-import/import/post-link.ts
new file mode 100644
index 00000000000..1237ada1e43
--- /dev/null
+++ b/ghost/core/core/server/services/content-import/import/post-link.ts
@@ -0,0 +1,16 @@
+import type { WrittenPost } from './post-repository';
+
+interface PostLinkOptions {
+ adminUrl: string;
+ publishedUrl(post: WrittenPost): string;
+}
+
+export function urlForImportedPost(post: WrittenPost, options: PostLinkOptions): string {
+ const data = post.toJSON();
+ if (data.status === 'draft') {
+ const editorType = data.type === 'page' ? 'page' : 'post';
+ return new URL(`#/editor/${editorType}/${post.id}`, options.adminUrl).href;
+ }
+
+ return options.publishedUrl(post);
+}
diff --git a/ghost/core/core/server/services/content-import/import/post-repository.ts b/ghost/core/core/server/services/content-import/import/post-repository.ts
index 4e2d46ef47d..eeb53e07bad 100644
--- a/ghost/core/core/server/services/content-import/import/post-repository.ts
+++ b/ghost/core/core/server/services/content-import/import/post-repository.ts
@@ -1,4 +1,5 @@
import type { PostData } from './post-data';
+import type { DuplicateMetadata } from './store';
import {
BookshelfPostRelationsResolver,
type PostRelationSource,
@@ -14,10 +15,11 @@ export interface WrittenPost {
export type PostWriteResult =
| { status: 'created'; post: WrittenPost; warnings: string[] }
| { status: 'updated'; post: WrittenPost; warnings: string[] }
- | { status: 'skipped'; reason: string };
+ | { status: 'skipped'; reason: string; duplicate: DuplicateMetadata };
export interface PostWriteMetadata extends PostRelationSource {
sourceUpdatedAt?: string;
+ runTagName?: string;
}
export interface PostsRepository {
@@ -54,8 +56,18 @@ export class BookshelfPostsRepository implements PostsRepository {
): Promise {
return this._models.Base.transaction(async (transacting) => {
const writeOptions = { ...options, transacting };
- const lookupOptions = { ...writeOptions, forUpdate: true };
- let existingMatch: { post: WrittenPost; duplicateReason: string } | undefined;
+ const lookupOptions = {
+ ...writeOptions,
+ forUpdate: true,
+ ...(metadata.runTagName ? { withRelated: ['tags'] } : {}),
+ };
+ let existingMatch:
+ | {
+ post: WrittenPost;
+ duplicateReason: string;
+ matchedBy: DuplicateMetadata['matchedBy'];
+ }
+ | undefined;
if (data.comment_id) {
const existing = await this._models.Post.findOne(
@@ -67,6 +79,7 @@ export class BookshelfPostsRepository implements PostsRepository {
existingMatch = {
post: existing,
duplicateReason: `A post with the source ID "${data.comment_id}" already exists.`,
+ matchedBy: 'source_id',
};
}
}
@@ -81,14 +94,19 @@ export class BookshelfPostsRepository implements PostsRepository {
existingMatch = {
post: existing,
duplicateReason: `A post with the slug "${data.slug}" already exists.`,
+ matchedBy: 'slug',
};
}
}
if (existingMatch) {
- const { post: existing, duplicateReason } = existingMatch;
+ const { post: existing, duplicateReason, matchedBy } = existingMatch;
+ const duplicate: DuplicateMetadata = {
+ origin: hasRunTag(existing, metadata.runTagName) ? 'this_import' : 'pre_existing',
+ matchedBy,
+ };
if (!metadata.sourceUpdatedAt) {
- return { status: 'skipped', reason: duplicateReason };
+ return { status: 'skipped', reason: duplicateReason, duplicate };
}
const incomingInstant = new Date(metadata.sourceUpdatedAt).getTime();
@@ -104,6 +122,7 @@ export class BookshelfPostsRepository implements PostsRepository {
return {
status: 'skipped',
reason: 'The existing post is newer than or as recent as the imported row.',
+ duplicate,
};
}
@@ -136,3 +155,21 @@ export class BookshelfPostsRepository implements PostsRepository {
});
}
}
+
+function hasRunTag(post: WrittenPost, runTagName?: string): boolean {
+ if (!runTagName) {
+ return false;
+ }
+
+ const tags = post.toJSON().tags;
+ return (
+ Array.isArray(tags) &&
+ tags.some(
+ (tag) =>
+ typeof tag === 'object' &&
+ tag !== null &&
+ 'name' in tag &&
+ (tag as { name?: unknown }).name === runTagName,
+ )
+ );
+}
diff --git a/ghost/core/core/server/services/content-import/import/reader.ts b/ghost/core/core/server/services/content-import/import/reader.ts
index d1b9100f2af..9c3bb410e68 100644
--- a/ghost/core/core/server/services/content-import/import/reader.ts
+++ b/ghost/core/core/server/services/content-import/import/reader.ts
@@ -1,14 +1,32 @@
-import { parse as parseCSV } from '../csv';
+import { parseWithSource, type Row } from '../csv';
import { EDITORIAL_POST_FIELDS, postImportRowSchema, type PostImportRow } from './row';
const FIELD_BY_HEADER: Record = Object.fromEntries(
EDITORIAL_POST_FIELDS.map((field) => [field, field]),
);
+export interface PreparedPostRow {
+ data: PostImportRow;
+ source?: Row;
+ line: number;
+}
+
+export interface PreparedPostRows {
+ columns: string[];
+ rows: PreparedPostRow[];
+}
+
export default async function readPostRows(
path: string,
mapping?: Record,
-): Promise {
- const rows = await parseCSV(path, mapping ?? FIELD_BY_HEADER);
- return rows.map((row) => postImportRowSchema.parse(row));
+): Promise {
+ const parsed = await parseWithSource(path, mapping ?? FIELD_BY_HEADER);
+ return {
+ columns: parsed.columns,
+ rows: parsed.rows.map(({ data, source, line }) => ({
+ data: postImportRowSchema.parse(data),
+ source,
+ line,
+ })),
+ };
}
diff --git a/ghost/core/core/server/services/content-import/import/report.ts b/ghost/core/core/server/services/content-import/import/report.ts
new file mode 100644
index 00000000000..f4619e81a59
--- /dev/null
+++ b/ghost/core/core/server/services/content-import/import/report.ts
@@ -0,0 +1,41 @@
+import { serialize } from '../csv';
+import type { ImportRun, RowOutcome } from './store';
+
+const REPORT_COLUMNS = [
+ 'line',
+ 'title',
+ 'outcome',
+ 'reason',
+ 'duplicate_origin',
+ 'matched_by',
+ 'warnings',
+ 'media_failures',
+ 'post_url',
+];
+
+function formatMediaFailures(row: RowOutcome): string {
+ return (
+ row.mediaFailures?.map(({ sourceUrl, reason }) => `${sourceUrl}: ${reason}`).join('\n') ?? ''
+ );
+}
+
+export default function buildImportReport(run: ImportRun): string | undefined {
+ if (!run.rows.length) {
+ return undefined;
+ }
+
+ return serialize(
+ run.rows.map((row) => ({
+ line: row.line,
+ title: row.title ?? '',
+ outcome: row.duplicate ? 'duplicate' : row.status,
+ reason: row.reason ?? '',
+ duplicate_origin: row.duplicate?.origin ?? '',
+ matched_by: row.duplicate?.matchedBy ?? '',
+ warnings: row.warnings?.join('\n') ?? '',
+ media_failures: formatMediaFailures(row),
+ post_url: row.url ?? '',
+ })),
+ { columns: REPORT_COLUMNS },
+ );
+}
diff --git a/ghost/core/core/server/services/content-import/import/schema.ts b/ghost/core/core/server/services/content-import/import/schema.ts
index bafd1601dff..b2e31d0ea36 100644
--- a/ghost/core/core/server/services/content-import/import/schema.ts
+++ b/ghost/core/core/server/services/content-import/import/schema.ts
@@ -26,6 +26,7 @@ export const importRequestSchema = z.object({
filePath: z.string().min(1),
fileName: z.string().min(1),
mapping: mappingSchema.optional(),
+ requestUserEmail: z.string().email().nullable().optional(),
});
export type ImportRequest = z.infer;
diff --git a/ghost/core/core/server/services/content-import/import/store.ts b/ghost/core/core/server/services/content-import/import/store.ts
index cd90b838ec7..e25d42445f2 100644
--- a/ghost/core/core/server/services/content-import/import/store.ts
+++ b/ghost/core/core/server/services/content-import/import/store.ts
@@ -2,12 +2,17 @@
// outcome per row. Nothing is persisted; the durable job system milestone
// replaces this.
-// skipped = the row was never attempted (the publisher can fix the file);
-// failed = row processing was attempted but did not produce a post.
+// skipped = an otherwise valid row required no write (for example a duplicate);
+// failed = the row could not produce a post, including source validation errors.
export type Clock = () => Date;
export type RowStatus = 'created' | 'updated' | 'skipped' | 'failed';
+export interface DuplicateMetadata {
+ origin: 'this_import' | 'pre_existing';
+ matchedBy: 'source_id' | 'slug';
+}
+
export interface RowOutcome {
// Source line number as a publisher sees it in a spreadsheet: the header is
// line 1, so the first data row is line 2.
@@ -15,10 +20,13 @@ export interface RowOutcome {
title: string | null;
status: RowStatus;
reason?: string;
+ duplicate?: DuplicateMetadata;
mediaFailures?: Array<{ sourceUrl: string; reason: string }>;
warnings?: string[];
postId?: string;
+ postType?: 'post' | 'page';
url?: string;
+ source?: Record;
}
export interface ImportRun {
@@ -28,6 +36,7 @@ export interface ImportRun {
finishedAt?: Date;
failureReason?: string;
total: number;
+ sourceColumns: string[];
rows: RowOutcome[];
}
@@ -44,7 +53,7 @@ export class ImportRunStore {
this._now = now;
}
- create(id: string, total: number): ImportRun {
+ create(id: string, total: number, sourceColumns: string[] = []): ImportRun {
this.evict();
const run: ImportRun = {
@@ -52,6 +61,7 @@ export class ImportRunStore {
status: 'running',
startedAt: this._now(),
total,
+ sourceColumns,
rows: [],
};
this._runs.set(id, run);
@@ -83,6 +93,10 @@ export class ImportRunStore {
return this._runs.get(id);
}
+ release(id: string): void {
+ this._runs.delete(id);
+ }
+
// A running run is never evicted, whatever its age: the job holds only the runId,
// so evicting mid-import would silently turn its record()/finish() calls into
// no-ops and lose the report. The count cap can briefly overshoot while several
diff --git a/ghost/core/core/server/services/content-import/index.ts b/ghost/core/core/server/services/content-import/index.ts
index 76d14d09aa2..5efde634691 100644
--- a/ghost/core/core/server/services/content-import/index.ts
+++ b/ghost/core/core/server/services/content-import/index.ts
@@ -1,5 +1,10 @@
import { z } from 'zod';
-import ContentCSVImporter, { type ImportAccepted, type FailureReporter } from './import/importer';
+import ContentCSVImporter, {
+ type ImportAccepted,
+ type FailureReporter,
+ type EmailNotifications,
+} from './import/importer';
+import buildCompletionEmail from './import/completion-email';
import { BookshelfPostsRepository } from './import/post-repository';
import readPostRows from './import/reader';
import { importRequestSchema, type ImportRequest } from './import/schema';
@@ -7,6 +12,7 @@ import { ImportRunStore } from './import/store';
import { prepareImportSource } from './import/source';
import { PostMediaInliner } from './import/media';
import { isLocalMediaUrl } from './import/local-media-url';
+import { urlForImportedPost } from './import/post-link';
// The request is built from HTTP upload metadata, so it is validated at the
// service boundary rather than trusted.
@@ -27,9 +33,12 @@ function makeImporter(): ContentCSVImporter {
const jobsService = require('../jobs');
const settingsCache = require('../../../shared/settings-cache');
const urlService = require('../url');
+ const urlUtils = require('../../../shared/url-utils').default;
const mediaInlinerService = require('../media-inliner');
const config = require('../../../shared/config');
const ObjectID = require('bson-objectid').default;
+ const { GhostMailer } = require('../mail');
+ const ghostMailer = new GhostMailer();
// Inline jobs never reach the job manager's Sentry handler, which is wired to the
// offloaded worker path only, so a throw here would be seen by nobody.
@@ -45,6 +54,12 @@ function makeImporter(): ContentCSVImporter {
}
};
+ const email: EmailNotifications = {
+ send: (run, recipient) =>
+ ghostMailer.send(buildCompletionEmail(run, recipient, urlUtils.urlFor('admin', true))),
+ getDefaultRecipient: async () => (await models.User.getOwnerUser()).get('email'),
+ };
+
return new ContentCSVImporter({
readRows: readPostRows,
prepareSource: prepareImportSource,
@@ -66,12 +81,19 @@ function makeImporter(): ContentCSVImporter {
],
}),
}),
+ email,
addJob: jobsService.addJob.bind(jobsService),
report,
store: new ImportRunStore(),
- // Degrades to the 404 URL for a post the URL service cannot route yet (e.g. a draft).
urlForPost: (post) =>
- urlService.getUrlForResource({ ...post.toJSON(), type: 'posts' }, { absolute: true }),
+ urlForImportedPost(post, {
+ adminUrl: urlUtils.urlFor('admin', true),
+ publishedUrl: (publishedPost) =>
+ urlService.getUrlForResource(
+ { ...publishedPost.toJSON(), type: 'posts' },
+ { absolute: true },
+ ),
+ }),
newRunId: () => new ObjectID().toHexString(),
getTimezone: () => timezoneSchema.parse(settingsCache.get('timezone')),
});
diff --git a/ghost/core/core/server/services/custom-redirects/redirect-config-parser.ts b/ghost/core/core/server/services/custom-redirects/redirect-config-parser.ts
index 0c7ccd01d65..843062bebf2 100644
--- a/ghost/core/core/server/services/custom-redirects/redirect-config-parser.ts
+++ b/ghost/core/core/server/services/custom-redirects/redirect-config-parser.ts
@@ -8,7 +8,7 @@ const messages = {
jsonInvalid: 'JSON input must be an array of redirect objects.',
yamlParse: 'Could not parse YAML: {context}.',
yamlInvalid: 'YAML input is invalid. Check the contents of your YAML file.',
- redirectsHelp: 'https://ghost.org/docs/themes/routing/#redirects',
+ redirectsHelp: 'https://docs.ghost.org/themes/routing/#redirects',
};
export const parseJson = (content: string): RedirectConfig[] => {
@@ -98,7 +98,7 @@ export const parseYaml = (content: string): RedirectConfig[] => {
// Round-trips through parseYaml to []. Avoids a "download โ save โ
// re-upload throws on empty body" footgun on fresh installs.
const EMPTY_DOWNLOAD_TEMPLATE =
- '# See https://ghost.org/docs/themes/routing/#redirects\n301: {}\n302: {}\n';
+ '# See https://docs.ghost.org/themes/routing/#redirects\n301: {}\n302: {}\n';
export const serializeToYaml = (redirects: RedirectConfig[]): string => {
const permanent: RedirectConfig[] = [];
diff --git a/ghost/core/core/server/services/custom-redirects/redirects-service.ts b/ghost/core/core/server/services/custom-redirects/redirects-service.ts
index 82f539d1e04..fb31c4d07ac 100644
--- a/ghost/core/core/server/services/custom-redirects/redirects-service.ts
+++ b/ghost/core/core/server/services/custom-redirects/redirects-service.ts
@@ -8,7 +8,7 @@ import { errify } from '../../../shared/errify';
const messages = {
redirectsRegister: 'Could not register custom redirects.',
- redirectsHelp: 'https://ghost.org/docs/themes/routing/#redirects',
+ redirectsHelp: 'https://docs.ghost.org/themes/routing/#redirects',
skippedInvalidRedirect: 'Skipped invalid redirect: {context}',
rejectedRedirect: 'Could not register redirect: {from} -> {to}',
allRedirectsSkipped:
diff --git a/ghost/core/core/server/services/custom-redirects/validation.js b/ghost/core/core/server/services/custom-redirects/validation.js
index 500b619ebbe..08f947c2d78 100644
--- a/ghost/core/core/server/services/custom-redirects/validation.js
+++ b/ghost/core/core/server/services/custom-redirects/validation.js
@@ -5,7 +5,7 @@ const errors = require('@tryghost/errors');
const messages = {
redirectsWrongFormat: 'Incorrect redirects file format.',
invalidRedirectsFromRegex: 'Incorrect RegEx in redirects file.',
- redirectsHelp: 'https://ghost.org/docs/themes/routing/#redirects',
+ redirectsHelp: 'https://docs.ghost.org/themes/routing/#redirects',
};
/**
diff --git a/ghost/core/core/server/services/email-address/email-address-parser.d.ts b/ghost/core/core/server/services/email-address/email-address-parser.d.ts
deleted file mode 100644
index abec3a9c872..00000000000
--- a/ghost/core/core/server/services/email-address/email-address-parser.d.ts
+++ /dev/null
@@ -1,11 +0,0 @@
-export interface EmailAddress {
- address: string;
- name?: string;
-}
-
-declare class EmailAddressParser {
- static parse(email: string): EmailAddress | null;
- static stringify(email: EmailAddress): string;
-}
-
-export default EmailAddressParser;
diff --git a/ghost/core/core/server/services/email-address/email-address-parser.js b/ghost/core/core/server/services/email-address/email-address-parser.js
deleted file mode 100644
index 90f41bf291e..00000000000
--- a/ghost/core/core/server/services/email-address/email-address-parser.js
+++ /dev/null
@@ -1,72 +0,0 @@
-const addressparser = require('nodemailer/lib/addressparser');
-
-/**
- * @typedef {Object} EmailAddress
- * @property {string} address - The email address
- * @property {string} [name] - Optional name associated with the email
- */
-
-module.exports = class EmailAddressParser {
- /**
- * Parse an email string into an EmailAddress object
- * @param {string} email - Email string to parse
- * @returns {EmailAddress|null} Parsed email or null if invalid
- */
- static parse(email) {
- if (!email || typeof email !== 'string' || !email.length) {
- return null;
- }
-
- const parsed = addressparser(email);
-
- if (parsed.length !== 1) {
- return null;
- }
- const first = parsed[0];
-
- // Check first has a group property
- if ('group' in first) {
- // Unsupported format
- return null;
- }
-
- return {
- address: first.address,
- name: first.name || undefined,
- };
- }
-
- /**
- * Convert an EmailAddress object to a string representation
- * @param {EmailAddress} email - Email object to stringify
- * @returns {string} String representation of the email
- */
- static stringify(email) {
- if (!email.name) {
- return email.address;
- }
-
- const escapedName = email.name.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
-
- /**
- * https://linear.app/ghost/issue/ONC-969
- *
- * Gmail will reject emails that contain certain Unicode characters.
- * There isn't a documented list of which characters, and the error
- * messages points us to https://support.google.com/mail/?p=BlockedMessage
- *
- * We've found that the following characters are problematic:
- * - โ
WHITE HEAVY CHECK MARK (U+2705)
- * - โ CHECK MARK (U+2713)
- * - โ HEAVY CHECK MARK (U+2714)
- * - โ BALLOT BOX WITH CHECK (U+2611)
- * - ๐ธ LIGHT CHECK MARK (U+1F5F8)
- *
- * We remove these characters from the name.
- */
- const nameCleanedForGmail = escapedName
- .replace(/[\u2705\u2713\u2714\u2611\u{1F5F8}]/gu, '')
- .trim();
- return `"${nameCleanedForGmail}" <${email.address}>`;
- }
-};
diff --git a/ghost/core/core/server/services/email-address/email-address-parser.ts b/ghost/core/core/server/services/email-address/email-address-parser.ts
new file mode 100644
index 00000000000..04521277ea6
--- /dev/null
+++ b/ghost/core/core/server/services/email-address/email-address-parser.ts
@@ -0,0 +1,71 @@
+import addressparser from 'nodemailer/lib/addressparser';
+
+export type EmailAddress = {
+ /** The email address */
+ address: string;
+ /** Optional name associated with the email */
+ name?: string;
+};
+
+/**
+ * Parse an email string into an EmailAddress object
+ * @param email Email string to parse
+ * @returns Parsed email or null if invalid
+ */
+export const parse = (email: string): null | EmailAddress => {
+ if (!email || typeof email !== 'string' || !email.length) {
+ return null;
+ }
+
+ const parsed = addressparser(email);
+
+ if (parsed.length !== 1) {
+ return null;
+ }
+ const first = parsed[0];
+
+ // Check first has a group property
+ if ('group' in first) {
+ // Unsupported format
+ return null;
+ }
+
+ return {
+ address: first.address,
+ name: first.name || undefined,
+ };
+};
+
+/**
+ * Convert an EmailAddress object to a string representation
+ * @param email Email object to stringify
+ * @returns String representation of the email
+ */
+export const stringify = (email: Readonly): string => {
+ if (!email.name) {
+ return email.address;
+ }
+
+ const escapedName = email.name.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
+
+ /**
+ * https://linear.app/ghost/issue/ONC-969
+ *
+ * Gmail will reject emails that contain certain Unicode characters.
+ * There isn't a documented list of which characters, and the error
+ * messages points us to https://support.google.com/mail/?p=BlockedMessage
+ *
+ * We've found that the following characters are problematic:
+ * - โ
WHITE HEAVY CHECK MARK (U+2705)
+ * - โ CHECK MARK (U+2713)
+ * - โ HEAVY CHECK MARK (U+2714)
+ * - โ BALLOT BOX WITH CHECK (U+2611)
+ * - ๐ธ LIGHT CHECK MARK (U+1F5F8)
+ *
+ * We remove these characters from the name.
+ */
+ const nameCleanedForGmail = escapedName
+ .replace(/[\u2705\u2713\u2714\u2611\u{1F5F8}]/gu, '')
+ .trim();
+ return `"${nameCleanedForGmail}" <${email.address}>`;
+};
diff --git a/ghost/core/core/server/services/email-address/email-address-service.ts b/ghost/core/core/server/services/email-address/email-address-service.ts
index fd5ae15fccc..623234f57d4 100644
--- a/ghost/core/core/server/services/email-address/email-address-service.ts
+++ b/ghost/core/core/server/services/email-address/email-address-service.ts
@@ -1,5 +1,7 @@
import logging from '@tryghost/logging';
-import EmailAddressParser, { EmailAddress } from './email-address-parser.js';
+import * as emailAddressParser from './email-address-parser';
+
+type EmailAddress = emailAddressParser.EmailAddress;
export type EmailAddresses = {
from: EmailAddress;
@@ -46,7 +48,7 @@ export class EmailAddressService {
if (!fallbackAddress) {
return null;
}
- return EmailAddressParser.parse(fallbackAddress);
+ return emailAddressParser.parse(fallbackAddress);
};
this.#isValidEmailAddress = dependencies.isValidEmailAddress;
}
@@ -82,8 +84,8 @@ export class EmailAddressService {
}
getAddressFromString(from: string, replyTo?: string): EmailAddresses {
- const parsedFrom = EmailAddressParser.parse(from);
- const parsedReplyTo = replyTo ? EmailAddressParser.parse(replyTo) : undefined;
+ const parsedFrom = emailAddressParser.parse(from);
+ const parsedReplyTo = replyTo ? emailAddressParser.parse(replyTo) : undefined;
return this.getAddress({
from: parsedFrom ?? this.defaultFromEmail,
diff --git a/ghost/core/core/server/services/email-analytics/index.ts b/ghost/core/core/server/services/email-analytics/index.ts
index a7476a0f4ec..c581799c971 100644
--- a/ghost/core/core/server/services/email-analytics/index.ts
+++ b/ghost/core/core/server/services/email-analytics/index.ts
@@ -28,7 +28,7 @@ import type * as AutomationsApi from '../automations/automations-api';
import { AutomationEmailAnalyticsBatchProcessor } from './automation-email-analytics-batch-processor';
import { GiftEmailAnalyticsBatchProcessor } from './gift-email-analytics-batch-processor';
import { StartGiftEmailAnalyticsJobEvent } from './events/start-gift-email-analytics-job-event';
-import { deliveryService as giftDeliveryService } from '../gifts';
+import type { GiftDeliveryService } from '../gifts/gift-delivery-service';
import { GIFT_DELIVERY_EMAIL_TAG } from '../gifts/constants';
export const newsletters = new EmailAnalyticsServiceWrapper({
@@ -49,6 +49,7 @@ export const init = ({
db,
domainEvents,
emailSuppressionList,
+ giftDeliveryService,
membersRepository,
models: { Email, EmailRecipientFailure, EmailSpamComplaintEvent },
metrics,
@@ -63,6 +64,7 @@ export const init = ({
db: { knex: Knex };
domainEvents: Pick;
emailSuppressionList: Pick;
+ giftDeliveryService: Pick;
membersRepository: Pick;
models: {
Email: Email;
@@ -182,11 +184,6 @@ export const init = ({
},
metrics,
settingsCache,
- createEventProcessor: () =>
- new GiftEmailAnalyticsBatchProcessor({
- giftDeliveryService: {
- recordOutcome: (data) => giftDeliveryService!.recordOutcome(data),
- },
- }),
+ createEventProcessor: () => new GiftEmailAnalyticsBatchProcessor({ giftDeliveryService }),
});
};
diff --git a/ghost/core/core/server/services/email-service/batch-sending-service.js b/ghost/core/core/server/services/email-service/batch-sending-service.js
index 56c0e097b65..03029bb8f25 100644
--- a/ghost/core/core/server/services/email-service/batch-sending-service.js
+++ b/ghost/core/core/server/services/email-service/batch-sending-service.js
@@ -33,7 +33,6 @@ class BatchSendingService {
#models;
#db;
#sentry;
- #debugStorageFilePath;
#getRequiredUrlRelations;
#shuttingDown = false;
#inFlight = new Set();
@@ -68,7 +67,6 @@ class BatchSendingService {
* @param {object} [dependencies.BEFORE_RETRY_CONFIG]
* @param {object} [dependencies.AFTER_RETRY_CONFIG]
* @param {object} [dependencies.MAILGUN_API_RETRY_CONFIG]
- * @param {string} [dependencies.debugStorageFilePath]
*/
constructor({
emailRenderer,
@@ -83,7 +81,6 @@ class BatchSendingService {
BEFORE_RETRY_CONFIG,
AFTER_RETRY_CONFIG,
MAILGUN_API_RETRY_CONFIG,
- debugStorageFilePath,
}) {
this.#emailRenderer = emailRenderer;
this.#sendingService = sendingService;
@@ -93,7 +90,6 @@ class BatchSendingService {
this.#models = models;
this.#db = db;
this.#sentry = sentry;
- this.#debugStorageFilePath = debugStorageFilePath;
this.#getRequiredUrlRelations = getRequiredUrlRelations;
if (BEFORE_RETRY_CONFIG) {
diff --git a/ghost/core/core/server/services/email-service/email-renderer.js b/ghost/core/core/server/services/email-service/email-renderer.js
index b2cfdcfe7ca..879e5f0fd74 100644
--- a/ghost/core/core/server/services/email-service/email-renderer.js
+++ b/ghost/core/core/server/services/email-service/email-renderer.js
@@ -13,7 +13,7 @@ function isUnsplashImage(url) {
}
const { DateTime } = require('luxon');
const htmlToPlaintext = require('@tryghost/html-to-plaintext');
-const EmailAddressParser = require('../email-address/email-address-parser');
+const emailAddressParser = require('../email-address/email-address-parser');
const { getEmailDesign } = require('../email-rendering/email-design');
const { registerHelpers } = require('./helpers/register-helpers');
const crypto = require('crypto');
@@ -337,7 +337,7 @@ class EmailRenderer {
}
#getRawFromAddress(post, newsletter) {
- // Pass the raw name through; EmailAddressParser.stringify() is the single
+ // Pass the raw name through; emailAddressParser.stringify() is the single
// point that escapes it for the RFC5322 quoted-string From header. Escaping
// here too would double-escape (e.g. a title containing a double quote).
let senderName = this.#settingsCache.get('title') || '';
@@ -394,7 +394,7 @@ class EmailRenderer {
{ useFallbackAddress },
);
- return EmailAddressParser.stringify(addresses.from);
+ return emailAddressParser.stringify(addresses.from);
}
/**
@@ -423,7 +423,7 @@ class EmailRenderer {
);
if (addresses.replyTo) {
- return EmailAddressParser.stringify(addresses.replyTo);
+ return emailAddressParser.stringify(addresses.replyTo);
}
return null;
}
diff --git a/ghost/core/core/server/services/email-service/email-service-wrapper.js b/ghost/core/core/server/services/email-service/email-service-wrapper.js
index e049e855e19..22d767d6157 100644
--- a/ghost/core/core/server/services/email-service/email-service-wrapper.js
+++ b/ghost/core/core/server/services/email-service/email-service-wrapper.js
@@ -1,5 +1,4 @@
const debug = require('@tryghost/debug')('i18n');
-const logging = require('@tryghost/logging');
const url = require('../../api/endpoints/utils/serializers/output/utils/url');
const events = require('../../lib/common/events');
@@ -55,12 +54,6 @@ class EmailServiceWrapper {
const emailAnalyticsJobs = require('../email-analytics/jobs');
const { cachedImageSizeFromUrl } = require('../../lib/image');
- // capture errors from mailgun client and log them in sentry
- const errorHandler = (error) => {
- logging.info(`Capturing error for mailgun email provider service`);
- sentry.captureException(error);
- };
-
// Mailgun client instance for email provider
const mailgunClient = new MailgunClient({
config: configService,
@@ -78,7 +71,6 @@ class EmailServiceWrapper {
const mailgunEmailProvider = new MailgunEmailProvider({
mailgunClient,
config: configService,
- errorHandler,
});
const emailRenderer = new EmailRenderer({
@@ -134,7 +126,6 @@ class EmailServiceWrapper {
db,
sentry,
getRequiredUrlRelations,
- debugStorageFilePath: configService.getContentPath('data'),
});
if (ghostServer) {
diff --git a/ghost/core/core/server/services/email-service/mailgun-email-provider.js b/ghost/core/core/server/services/email-service/mailgun-email-provider.js
index 105cd74b698..6b2cebc36e3 100644
--- a/ghost/core/core/server/services/email-service/mailgun-email-provider.js
+++ b/ghost/core/core/server/services/email-service/mailgun-email-provider.js
@@ -35,18 +35,15 @@ const debug = require('@tryghost/debug')('email-service:mailgun-provider-service
class MailgunEmailProvider {
#mailgunClient;
#config;
- #errorHandler;
/**
* @param {object} dependencies
* @param {import('../lib/mailgun-client')} dependencies.mailgunClient - mailgun client to send emails
* @param {Config} dependencies.config
- * @param {Function} [dependencies.errorHandler] - custom error handler for logging exceptions
*/
- constructor({ mailgunClient, config, errorHandler }) {
+ constructor({ mailgunClient, config }) {
this.#mailgunClient = mailgunClient;
this.#config = config;
- this.#errorHandler = errorHandler;
}
#createRecipientData(replacements) {
@@ -165,7 +162,7 @@ class MailgunEmailProvider {
message: this.#createMailgunErrorMessage(error),
errorDetails: JSON.stringify({ error, messageData }),
context: `Mailgun Error ${error.status}: ${error.details}`,
- help: `https://ghost.org/docs/newsletters/#bulk-email-configuration`,
+ help: `https://docs.ghost.org/newsletters/#bulk-email-configuration`,
code: 'BULK_EMAIL_SEND_FAILED',
});
} else {
diff --git a/ghost/core/core/server/services/gifts/index.ts b/ghost/core/core/server/services/gifts/index.ts
index 8801373e350..afa5feea001 100644
--- a/ghost/core/core/server/services/gifts/index.ts
+++ b/ghost/core/core/server/services/gifts/index.ts
@@ -28,7 +28,7 @@ export let deliveryService: GiftDeliveryService | undefined;
// of boot-time rebuilds (same contract post scheduling honours in boot.js).
let rescheduleDeliveriesOnBoot = true;
-export async function init(options: GiftServiceInitOptions): Promise {
+export function init(options: GiftServiceInitOptions): void {
if (service) {
return;
}
@@ -58,7 +58,7 @@ export async function init(options: GiftServiceInitOptions): Promise {
const labsService = require('../../../shared/labs');
const urlUtils = require('../../../shared/url-utils').default;
const settingsHelpers = require('../settings-helpers');
- const EmailAddressParser = require('../email-address/email-address-parser');
+ const emailAddressParser = require('../email-address/email-address-parser');
const { blogIcon } = require('../../../server/lib/image');
const { t } = require('../i18n');
@@ -80,7 +80,7 @@ export async function init(options: GiftServiceInitOptions): Promise {
bulkMailer: new MailgunClient({ config, settings: settingsCache }),
settingsCache,
urlUtils,
- getFromAddress: () => EmailAddressParser.stringify(settingsHelpers.getDefaultEmail()),
+ getFromAddress: () => emailAddressParser.stringify(settingsHelpers.getDefaultEmail()),
getReplyToAddress: () => settingsHelpers.getMembersSupportAddress(),
blogIcon,
t,
diff --git a/ghost/core/core/server/services/indexnow-ping/indexnow-ping-service.ts b/ghost/core/core/server/services/indexnow-ping/indexnow-ping-service.ts
index c1be926ceb3..caa0de1b5ff 100644
--- a/ghost/core/core/server/services/indexnow-ping/indexnow-ping-service.ts
+++ b/ghost/core/core/server/services/indexnow-ping/indexnow-ping-service.ts
@@ -265,7 +265,7 @@ export class IndexNowPingService {
err,
message: err.message,
context: tpl(messages.requestFailedError, { service: 'IndexNow' }),
- help: tpl(messages.requestFailedHelp, { url: 'https://ghost.org/docs/' }),
+ help: tpl(messages.requestFailedHelp, { url: 'https://docs.ghost.org/' }),
}),
};
}
@@ -288,7 +288,7 @@ export class IndexNowPingService {
err,
message: err.message,
context: tpl(messages.requestFailedError, { service: 'IndexNow' }),
- help: tpl(messages.requestFailedHelp, { url: 'https://ghost.org/docs/' }),
+ help: tpl(messages.requestFailedHelp, { url: 'https://docs.ghost.org/' }),
}),
};
}
diff --git a/ghost/core/core/server/services/jobs-service/register-job-handlers.ts b/ghost/core/core/server/services/jobs-service/register-job-handlers.ts
index 932b5a1f87d..9e1451df159 100644
--- a/ghost/core/core/server/services/jobs-service/register-job-handlers.ts
+++ b/ghost/core/core/server/services/jobs-service/register-job-handlers.ts
@@ -2,6 +2,8 @@ import errors from '@tryghost/errors';
import { JobsService } from './jobs-service';
import CleanTokensJob from '../members/jobs/clean-tokens-job';
import cleanTokens from '../members/jobs/clean-tokens-task';
+import CleanExpiredCompedJob from '../members/jobs/clean-expired-comped-job';
+import cleanExpiredComped from '../members/jobs/clean-expired-comped-task';
import * as gifts from '../gifts';
import CleanGiftsJob from '../gifts/jobs/clean-gifts-job';
import ExternalMediaInliner from '../media-inliner/external-media-inliner';
@@ -11,6 +13,18 @@ interface RegisterJobHandlersDependencies {
jobsService: JobsService;
db: typeof import('../../data/db');
logging: typeof import('@tryghost/logging');
+ // Structural types: models, lib/common/events and shared/sentry are
+ // untyped CommonJS modules, so name just the surface the handlers consume.
+ models: {
+ Member: {
+ findOne(
+ data: Record,
+ options: Record,
+ ): Promise<{ attributes: Record }>;
+ };
+ };
+ events: { emit(name: string, model: unknown, options: Record): void };
+ sentry: { captureException(err: unknown): void };
mediaInliner: ExternalMediaInliner;
}
@@ -18,12 +32,19 @@ export default function registerJobHandlers({
jobsService,
db,
logging,
+ models,
+ events,
+ sentry,
mediaInliner,
}: RegisterJobHandlersDependencies): void {
jobsService.handle(CleanTokensJob, async () => {
await cleanTokens({ db, logging });
});
+ jobsService.handle(CleanExpiredCompedJob, async () => {
+ await cleanExpiredComped({ db, models, events, logging, sentry });
+ });
+
jobsService.handle(CleanGiftsJob, async () => {
if (!gifts.service) {
throw new errors.IncorrectUsageError({
diff --git a/ghost/core/core/server/services/jobs/job-service.js b/ghost/core/core/server/services/jobs/job-service.js
index adbc2b23976..c149b8cbe1a 100644
--- a/ghost/core/core/server/services/jobs/job-service.js
+++ b/ghost/core/core/server/services/jobs/job-service.js
@@ -9,22 +9,13 @@ const models = require('../../models');
const sentry = require('../../../shared/sentry');
const domainEvents = require('@tryghost/domain-events');
const config = require('../../../shared/config');
-const WorkerModelEventBridge = require('./worker-model-event-bridge');
const errorHandler = (error, workerMeta) => {
logging.error(error, `[Background Job] ${workerMeta.name} failed`);
sentry.captureException(error);
};
const events = require('../../lib/common/events');
-const workerModelEventBridge = new WorkerModelEventBridge({ models, events, logging, sentry });
const workerMessageHandler = ({ name, message }) => {
- if (workerModelEventBridge.isModelEventMessage(message)) {
- // Carries its own `eventName` rather than job-manager's reserved `event` key,
- // so it routes through the bridge instead of being dispatched as a raw domain event.
- workerModelEventBridge.handle(message, { jobName: name });
- return;
- }
-
if (typeof message === 'string' && !['done', 'cancelled'].includes(message)) {
logging.info(`[Background Job] ${name}: ${message}`);
}
diff --git a/ghost/core/core/server/services/jobs/worker-model-event-bridge.js b/ghost/core/core/server/services/jobs/worker-model-event-bridge.js
deleted file mode 100644
index 8c4fdb74da9..00000000000
--- a/ghost/core/core/server/services/jobs/worker-model-event-bridge.js
+++ /dev/null
@@ -1,149 +0,0 @@
-const { z } = require('zod');
-const moment = require('moment');
-
-const MODEL_EVENT_TYPE = 'model-event';
-const SUPPORTED_MODEL_EVENTS = {
- 'member.edited': {
- model: 'Member',
- },
-};
-
-const attributesSchema = z.record(z.string(), z.unknown());
-
-const messageSchema = z
- .object({
- type: z.literal(MODEL_EVENT_TYPE),
- eventName: z.enum(Object.keys(SUPPORTED_MODEL_EVENTS)),
- model: z.string(),
- id: z.string().min(1),
- previous: attributesSchema,
- changed: attributesSchema.refine((value) => Object.keys(value).length > 0, {
- message: 'must include at least one changed attribute',
- }),
- options: attributesSchema.optional(),
- })
- .refine((message) => SUPPORTED_MODEL_EVENTS[message.eventName]?.model === message.model, {
- message: 'does not match the event',
- path: ['model'],
- });
-
-class WorkerModelEventBridge {
- constructor({ models, events, logging, sentry }) {
- this.models = models;
- this.events = events;
- this.logging = logging;
- this.sentry = sentry;
- }
-
- isModelEventMessage(message) {
- return isObject(message) && message.type === MODEL_EVENT_TYPE;
- }
-
- async handle(message, meta = {}) {
- const validationError = this.validate(message);
-
- if (validationError) {
- this.logging.warn(
- `Ignoring invalid worker model event from job ${meta.jobName || 'unknown'}: ${validationError}`,
- );
- return false;
- }
-
- try {
- return await this.emitModelEvent(message);
- } catch (err) {
- this.logging.error(err);
- this.sentry.captureException(err);
- return false;
- }
- }
-
- validate(message) {
- const result = messageSchema.safeParse(message);
-
- if (result.success) {
- return null;
- }
-
- return result.error.issues
- .map((issue) =>
- issue.path.length ? `${issue.path.join('.')}: ${issue.message}` : issue.message,
- )
- .join('; ');
- }
-
- async emitModelEvent(message) {
- const Model = this.models[message.model];
- let model;
-
- try {
- model = await Model.findOne(
- {
- id: message.id,
- },
- {
- require: true,
- context: { internal: true },
- },
- );
- } catch (err) {
- if (isNotFoundError(err)) {
- this.logging.warn(
- `Could not emit worker model event ${message.eventName}: ${message.model} ${message.id} was not found`,
- );
- return false;
- }
-
- throw err;
- }
-
- model._previousAttributes = normalizeDates({
- ...model.attributes,
- ...message.previous,
- });
- model._changed = normalizeDates(message.changed);
-
- const options = normalizeOptions(message.options);
- this.events.emit(message.eventName, model, options);
- return true;
- }
-}
-
-function normalizeOptions(options = {}) {
- return {
- ...options,
- context: {
- ...options.context,
- internal: true,
- },
- };
-}
-
-function normalizeDates(attributes) {
- const normalized = { ...attributes };
-
- for (const [key, value] of Object.entries(normalized)) {
- if (key.endsWith('_at') && value && !(value instanceof Date)) {
- normalized[key] = moment.utc(value).toDate();
- }
- }
-
- return normalized;
-}
-
-function isObject(value) {
- return !!value && typeof value === 'object' && !Array.isArray(value);
-}
-
-function isNotFoundError(err) {
- return (
- err &&
- (err.errorType === 'NotFoundError' ||
- err.name === 'NotFoundError' ||
- err.message === 'NotFound' ||
- err.message === 'EmptyResponse')
- );
-}
-
-module.exports = WorkerModelEventBridge;
-module.exports.MODEL_EVENT_TYPE = MODEL_EVENT_TYPE;
diff --git a/ghost/core/core/server/services/mail/ghost-mailer.js b/ghost/core/core/server/services/mail/ghost-mailer.js
index 805fd4a59b8..026d86be0f6 100644
--- a/ghost/core/core/server/services/mail/ghost-mailer.js
+++ b/ghost/core/core/server/services/mail/ghost-mailer.js
@@ -17,7 +17,7 @@ const messages = {
reason: ' Reason: {reason}.',
messageSent: 'Message sent. Double check inbox and spam folder!',
};
-const EmailAddressParser = require('../email-address/email-address-parser');
+const emailAddressParser = require('../email-address/email-address-parser');
const DEFAULT_TAGS = ['ghost-email', 'transactional-email'];
const MAX_MAILGUN_TAGS = 10;
@@ -54,8 +54,8 @@ function getFromAddress(requestedFromAddress, requestedReplyToAddress) {
}
return {
- from: EmailAddressParser.stringify(addresses.from),
- replyTo: addresses.replyTo ? EmailAddressParser.stringify(addresses.replyTo) : null,
+ from: emailAddressParser.stringify(addresses.from),
+ replyTo: addresses.replyTo ? emailAddressParser.stringify(addresses.replyTo) : null,
};
}
@@ -100,7 +100,7 @@ function createMessage(message) {
*/
function createMailError({ message, err, ignoreDefaultMessage } = { message: '' }) {
const helpMessage = tpl(messages.checkEmailConfigInstructions, {
- url: 'https://ghost.org/docs/config/#mail',
+ url: 'https://docs.ghost.org/config/#mail',
});
const defaultErrorMessage = tpl(messages.failedSendingEmailError);
diff --git a/ghost/core/core/server/services/member-welcome-emails/service.js b/ghost/core/core/server/services/member-welcome-emails/service.js
index b8fbe49a2b7..1481ebab694 100644
--- a/ghost/core/core/server/services/member-welcome-emails/service.js
+++ b/ghost/core/core/server/services/member-welcome-emails/service.js
@@ -7,7 +7,7 @@ const MagicLink = require('../lib/magic-link/magic-link');
const sentry = require('../../../shared/sentry');
const emailAddressService = require('../email-address');
const settingsHelpers = require('../settings-helpers');
-const EmailAddressParser = require('../email-address/email-address-parser');
+const emailAddressParser = require('../email-address/email-address-parser');
const mail = require('../mail');
const MailgunClient = require('../lib/mailgun-client');
const config = require('../../../shared/config');
@@ -152,7 +152,7 @@ class MemberWelcomeEmailService {
},
});
- const from = EmailAddressParser.stringify(fromAddresses.from);
+ const from = emailAddressParser.stringify(fromAddresses.from);
const replyToSetting = newsletter.get('sender_reply_to');
let replyTo = null;
@@ -173,7 +173,7 @@ class MemberWelcomeEmailService {
});
if (addresses.replyTo) {
- replyTo = EmailAddressParser.stringify(addresses.replyTo);
+ replyTo = emailAddressParser.stringify(addresses.replyTo);
}
}
@@ -199,10 +199,10 @@ class MemberWelcomeEmailService {
async #getEffectiveSenderOptions(automatedSender = {}) {
const defaultOptions = await this.#getSenderOptions();
const defaultFrom =
- EmailAddressParser.parse(defaultOptions.from || '') ||
+ emailAddressParser.parse(defaultOptions.from || '') ||
emailAddressService.service.defaultFromEmail;
const defaultReplyTo = defaultOptions.replyTo
- ? EmailAddressParser.parse(defaultOptions.replyTo)
+ ? emailAddressParser.parse(defaultOptions.replyTo)
: undefined;
const senderName = trimValue(automatedSender.senderName) || defaultFrom?.name || undefined;
@@ -218,10 +218,10 @@ class MemberWelcomeEmailService {
});
return {
- from: EmailAddressParser.stringify(addresses.from),
+ from: emailAddressParser.stringify(addresses.from),
...(addresses.replyTo
? {
- replyTo: EmailAddressParser.stringify(addresses.replyTo),
+ replyTo: emailAddressParser.stringify(addresses.replyTo),
}
: {}),
};
diff --git a/ghost/core/core/server/services/members/jobs/clean-expired-comped-job.ts b/ghost/core/core/server/services/members/jobs/clean-expired-comped-job.ts
new file mode 100644
index 00000000000..03940ffbbe1
--- /dev/null
+++ b/ghost/core/core/server/services/members/jobs/clean-expired-comped-job.ts
@@ -0,0 +1,5 @@
+import { Job } from '../../jobs-service/job';
+
+export default class CleanExpiredCompedJob extends Job {
+ static type = 'clean-expired-comped';
+}
diff --git a/ghost/core/core/server/services/members/jobs/clean-expired-comped-task.ts b/ghost/core/core/server/services/members/jobs/clean-expired-comped-task.ts
new file mode 100644
index 00000000000..8cfeadf655b
--- /dev/null
+++ b/ghost/core/core/server/services/members/jobs/clean-expired-comped-task.ts
@@ -0,0 +1,245 @@
+import type { Knex } from 'knex';
+
+const ObjectId = require('bson-objectid').default;
+const { chunk: chunkArray } = require('lodash');
+const moment = require('moment');
+
+interface MemberRow {
+ id: string;
+ status: string;
+ updated_at: Date | string;
+}
+
+interface ExpiredCompedRow {
+ id: string;
+ member_id: string;
+}
+
+interface MemberModel {
+ attributes: Record;
+ _previousAttributes?: Record;
+ _changed?: Record;
+}
+
+interface CleanExpiredCompedDeps {
+ db: { knex: Knex };
+ models: {
+ Member: {
+ findOne(
+ data: Record,
+ options: Record,
+ ): Promise;
+ };
+ };
+ events: { emit(name: string, model: MemberModel, options: Record): void };
+ logging: {
+ info(...args: unknown[]): void;
+ warn(...args: unknown[]): void;
+ error(...args: unknown[]): void;
+ };
+ sentry: { captureException(err: unknown): void };
+}
+
+export interface CleanExpiredCompedResult {
+ deletedSubscriptionCount: number;
+ updatedMemberCount: number;
+ emittedEventCount: number;
+}
+
+// Removes expired comped entries from the members_products table, moves the
+// affected members from comped to free and records a comped->free status
+// event for each of them. The three writes commit or roll back together, so a
+// failed run leaves the source rows in place and a repeat run redoes the whole
+// unit of work instead of finishing a half-applied one.
+async function cleanExpiredComped({
+ db,
+ models,
+ events,
+ logging,
+ sentry,
+}: CleanExpiredCompedDeps): Promise {
+ const startedAt = Date.now();
+
+ const { deletedSubscriptionCount, updatedMemberCount, updatedMembers, now } =
+ await db.knex.transaction(async (trx) => {
+ const expiredCompedRows: ExpiredCompedRow[] = await trx('members_products')
+ // we need to be careful about the type here. .format() is the only
+ // thing that works across SQLite and MySQL: an ISO cutoff compares
+ // lexically against SQLite's 'YYYY-MM-DD HH:mm:ss' text and its 'T'
+ // sorts above ' ', wrongly matching same-UTC-day expiries
+ .where('expiry_at', '<', moment.utc().startOf('day').format('YYYY-MM-DD HH:mm:ss'))
+ .select('*');
+
+ if (!expiredCompedRows.length) {
+ return {
+ deletedSubscriptionCount: 0,
+ updatedMemberCount: 0,
+ updatedMembers: [] as MemberRow[],
+ now: new Date(),
+ };
+ }
+
+ const rowIds = expiredCompedRows.map((d) => d.id);
+ const memberIds = expiredCompedRows.map((d) => d.member_id);
+
+ // Only members still in the comped status change; anyone already paid,
+ // free or deleted is left untouched. The rows are locked for the rest
+ // of the transaction so a concurrent status change cannot slip between
+ // this read and the update below (forUpdate is a no-op on SQLite,
+ // where the single-writer transaction already guarantees this). The
+ // members rows are locked BEFORE the members_products delete so this
+ // transaction acquires locks in the same members -> members_products
+ // order as the member-edit flows (Bookshelf edit, linkSubscription);
+ // the reverse order could deadlock against them.
+ const compedMembers: MemberRow[] = await trx('members')
+ .whereIn('id', memberIds)
+ .andWhere('status', 'comped')
+ .forUpdate();
+
+ const deletedCount = await trx('members_products').whereIn('id', rowIds).del();
+
+ const updateMemberIds = compedMembers.map((d) => d.id);
+ const updatedAt = new Date();
+
+ const updatedCount = await trx('members').whereIn('id', updateMemberIds).update({
+ status: 'free',
+ updated_at: updatedAt,
+ });
+
+ const statusEvents = compedMembers.map((member) => {
+ return {
+ id: ObjectId().toHexString(),
+ member_id: member.id,
+ from_status: member.status,
+ to_status: 'free',
+ created_at: updatedAt,
+ };
+ });
+
+ // SQLite >= 3.32.0 can support 32766 host parameters
+ // each row uses 5 variables so โ32766/5โ = 6553
+ const chunkSize = 6553;
+
+ for (const chunk of chunkArray(statusEvents, chunkSize)) {
+ await trx('members_status_events').insert(chunk);
+ }
+
+ return {
+ deletedSubscriptionCount: deletedCount,
+ updatedMemberCount: updatedCount,
+ updatedMembers: compedMembers,
+ now: updatedAt,
+ };
+ });
+
+ // Emitted after the commit so listeners never observe state that could
+ // still roll back. A failed emission is logged and skipped rather than
+ // failing the job: the database changes are already durable and the
+ // emitted_event_count below makes the shortfall visible.
+ let emittedEventCount = 0;
+ for (const member of updatedMembers) {
+ const emitted = await emitMemberEditedEvent({ member, now, models, events, logging, sentry });
+ if (emitted) {
+ emittedEventCount += 1;
+ }
+ }
+
+ logging.info(
+ {
+ system: {
+ event: 'clean_expired_comped.completed',
+ deleted_subscription_count: deletedSubscriptionCount,
+ updated_member_count: updatedMemberCount,
+ emitted_event_count: emittedEventCount,
+ duration_ms: Date.now() - startedAt,
+ },
+ },
+ `[Background Job] clean-expired-comped removed ${deletedSubscriptionCount} expired subscriptions and updated ${updatedMemberCount} members`,
+ );
+
+ return { deletedSubscriptionCount, updatedMemberCount, emittedEventCount };
+}
+
+interface EmitMemberEditedEventOptions {
+ member: MemberRow;
+ now: Date;
+ models: CleanExpiredCompedDeps['models'];
+ events: CleanExpiredCompedDeps['events'];
+ logging: CleanExpiredCompedDeps['logging'];
+ sentry: CleanExpiredCompedDeps['sentry'];
+}
+
+// Emits member.edited in the shape its listeners (webhooks in particular)
+// expect: a freshly loaded model whose _previousAttributes carry the
+// pre-cleanup status and _changed the comped->free transition, with an
+// internal context.
+async function emitMemberEditedEvent({
+ member,
+ now,
+ models,
+ events,
+ logging,
+ sentry,
+}: EmitMemberEditedEventOptions): Promise {
+ try {
+ const model = await models.Member.findOne(
+ { id: member.id },
+ { require: true, context: { internal: true } },
+ );
+
+ model._previousAttributes = normalizeDates({
+ ...model.attributes,
+ status: member.status,
+ updated_at: member.updated_at,
+ });
+ model._changed = normalizeDates({
+ status: 'free',
+ updated_at: now,
+ });
+
+ events.emit('member.edited', model, { context: { internal: true } });
+ return true;
+ } catch (err) {
+ if (isNotFoundError(err)) {
+ logging.warn(
+ `Could not emit member.edited for expired comped member: Member ${member.id} was not found`,
+ );
+ return false;
+ }
+
+ logging.error(err);
+ sentry.captureException(err);
+ return false;
+ }
+}
+
+function normalizeDates(attributes: Record): Record {
+ const normalized = { ...attributes };
+
+ for (const [key, value] of Object.entries(normalized)) {
+ if (key.endsWith('_at') && value && !(value instanceof Date)) {
+ normalized[key] = moment.utc(value).toDate();
+ }
+ }
+
+ return normalized;
+}
+
+function isNotFoundError(err: unknown): boolean {
+ if (!err || typeof err !== 'object') {
+ return false;
+ }
+
+ const errorType = 'errorType' in err ? err.errorType : undefined;
+ const name = 'name' in err ? err.name : undefined;
+ const message = 'message' in err ? err.message : undefined;
+
+ return (
+ errorType === 'NotFoundError' ||
+ name === 'NotFoundError' ||
+ message === 'NotFound' ||
+ message === 'EmptyResponse'
+ );
+}
+
+export default cleanExpiredComped;
diff --git a/ghost/core/core/server/services/members/jobs/clean-expired-comped.js b/ghost/core/core/server/services/members/jobs/clean-expired-comped.js
deleted file mode 100644
index a5740f7b044..00000000000
--- a/ghost/core/core/server/services/members/jobs/clean-expired-comped.js
+++ /dev/null
@@ -1,132 +0,0 @@
-const { parentPort } = require('worker_threads');
-const ObjectId = require('bson-objectid').default;
-const { chunk: chunkArray } = require('lodash');
-const debug = require('@tryghost/debug')('jobs:clean-expired-comped');
-const moment = require('moment');
-
-// recurring job to clean expired complimentary subscriptions
-
-// Exit early when cancelled to prevent stalling shutdown. No cleanup needed when cancelling as everything is idempotent and will pick up
-// where it left off on next run
-function cancel() {
- if (parentPort) {
- parentPort.postMessage('cancelled before completion');
- parentPort.postMessage('cancelled');
- } else {
- setTimeout(() => {
- process.exit(0);
- }, 1000);
- }
-}
-
-if (parentPort) {
- parentPort.once('message', (message) => {
- if (message === 'cancel') {
- return cancel();
- }
- });
-}
-
-(async () => {
- const cleanupStartDate = new Date();
- const db = require('../../../data/db');
- if (parentPort) {
- parentPort.postMessage('execution started');
- }
- debug(`Starting cleanup of expired comp subscriptions`);
- const expiredCompedRows = await db
- .knex('members_products')
- .where('expiry_at', '<', moment.utc().startOf('day').toISOString())
- .select('*');
-
- let deletedExpiredSubs = 0;
- let updatedMembers = 0;
-
- // Run cleanup for expired comp subscriptions
- // Removes expired comped entries from members_products table
- // Updates affected members status to free from comped
- // Adds member status event for going from comped to free
- if (expiredCompedRows?.length) {
- const rowIds = expiredCompedRows.map((d) => d.id);
- const memberIds = expiredCompedRows.map((d) => d.member_id);
-
- // Delete all expired comped rows
- deletedExpiredSubs = await db.knex('members_products').whereIn('id', rowIds).del();
-
- // fetch all comped members to update
- const membersToUpdate = await db
- .knex('members')
- .whereIn('id', memberIds)
- .andWhere('status', 'comped');
-
- const updateMemberIds = membersToUpdate.map((d) => d.id);
- const now = new Date();
-
- // Update all comped members to free
- updatedMembers = await db.knex('members').whereIn('id', updateMemberIds).update({
- status: 'free',
- updated_at: now,
- });
-
- const statusEvents = membersToUpdate.map((member) => {
- return {
- id: ObjectId().toHexString(),
- member_id: member.id,
- from_status: member.status,
- to_status: 'free',
- created_at: now,
- };
- });
-
- // SQLite >= 3.32.0 can support 32766 host parameters
- // each row uses 5 variables so โ32766/5โ = 6553
- const chunkSize = 6553;
-
- const chunks = chunkArray(statusEvents, chunkSize);
-
- // Adds status event for members going comped->free
- for (const chunk of chunks) {
- await db.knex('members_status_events').insert(chunk);
- }
-
- if (parentPort) {
- for (const member of membersToUpdate) {
- parentPort.postMessage({
- type: 'model-event',
- eventName: 'member.edited',
- model: 'Member',
- id: member.id,
- previous: {
- status: member.status,
- updated_at: member.updated_at,
- },
- changed: {
- status: 'free',
- updated_at: now,
- },
- options: {
- context: { internal: true },
- },
- });
- }
- }
- }
-
- let cleanupEndDate = new Date();
-
- debug(
- `Removed ${deletedExpiredSubs} expired subscriptions, updated ${updatedMembers} members in ${cleanupEndDate.valueOf() - cleanupStartDate.valueOf()}ms`,
- );
-
- if (parentPort) {
- parentPort.postMessage(
- `completed in ${cleanupEndDate.valueOf() - cleanupStartDate.valueOf()}ms: removed ${deletedExpiredSubs} expired subscriptions and updated ${updatedMembers} members`,
- );
- parentPort.postMessage('done');
- } else {
- // give the logging pipes time finish writing before exit
- setTimeout(() => {
- process.exit(0);
- }, 1000);
- }
-})();
diff --git a/ghost/core/core/server/services/members/jobs/index.js b/ghost/core/core/server/services/members/jobs/index.js
index a3e56730ec3..617a0ffd5d1 100644
--- a/ghost/core/core/server/services/members/jobs/index.js
+++ b/ghost/core/core/server/services/members/jobs/index.js
@@ -1,7 +1,6 @@
-const path = require('path');
const logging = require('@tryghost/logging');
-const jobsService = require('../../jobs');
const CleanTokensJob = require('./clean-tokens-job').default;
+const CleanExpiredCompedJob = require('./clean-expired-comped-job').default;
let hasScheduled = {
expiredComped: false,
@@ -19,32 +18,19 @@ function randomDailyCron(maxHour = 24) {
return `${s} ${m} ${h} * * *`;
}
-function scheduleJob(key, name, jobFile, maxHour = 6) {
- if (alreadyScheduledOrTest(key)) {
- return hasScheduled[key];
- }
-
- const s = Math.floor(Math.random() * 60);
- const m = Math.floor(Math.random() * 60);
- const h = Math.floor(Math.random() * maxHour);
-
- const at = `${s} ${m} ${h} * * *`;
-
- logging.info(`[Background Job] ${name} scheduled at ${at}`);
- jobsService.addJob({
- at,
- job: path.resolve(__dirname, jobFile),
- name,
- });
-
- hasScheduled[key] = true;
-
- return true;
-}
-
module.exports = {
async scheduleExpiredCompCleanupJob() {
- return scheduleJob('expiredComped', 'clean-expired-comped', 'clean-expired-comped.js');
+ if (alreadyScheduledOrTest('expiredComped')) {
+ return;
+ }
+
+ const classBasedJobs = require('../../jobs-service').getInstance();
+ // Keep the legacy off-peak window: a random time between 00:00 and 05:59
+ const cron = randomDailyCron(6);
+ logging.info(`[Background Job] clean-expired-comped scheduled at ${cron}`);
+ await classBasedJobs.scheduleRecurring(new CleanExpiredCompedJob(), { cron });
+
+ hasScheduled.expiredComped = true;
},
async scheduleTokenCleanupJob() {
diff --git a/ghost/core/core/server/services/members/members-api/controllers/router-controller.js b/ghost/core/core/server/services/members/members-api/controllers/router-controller.js
index 59c0f663736..d23315468ea 100644
--- a/ghost/core/core/server/services/members/members-api/controllers/router-controller.js
+++ b/ghost/core/core/server/services/members/members-api/controllers/router-controller.js
@@ -140,7 +140,6 @@ module.exports = class RouterController {
* @param {any} deps.offersAPI
* @param {any} deps.paymentsService
* @param {any} deps.memberRepository
- * @param {any} deps.StripePrice
* @param {() => boolean} deps.allowSelfSignup
* @param {any} deps.magicLinkService
* @param {import('../../../stripe/stripe-api')} deps.stripeAPIService
@@ -160,7 +159,6 @@ module.exports = class RouterController {
paymentsService,
tiersService,
memberRepository,
- StripePrice,
allowSelfSignup,
magicLinkService,
stripeAPIService,
@@ -180,7 +178,6 @@ module.exports = class RouterController {
this._paymentsService = paymentsService;
this._tiersService = tiersService;
this._memberRepository = memberRepository;
- this._StripePrice = StripePrice;
this._allowSelfSignup = allowSelfSignup;
this._magicLinkService = magicLinkService;
this._stripeAPIService = stripeAPIService;
diff --git a/ghost/core/core/server/services/members/members-api/members-api.js b/ghost/core/core/server/services/members/members-api/members-api.js
index 7c429d31f33..cf70a5b2cb6 100644
--- a/ghost/core/core/server/services/members/members-api/members-api.js
+++ b/ghost/core/core/server/services/members/members-api/members-api.js
@@ -41,7 +41,6 @@ module.exports = function MembersAPI({
MemberProductEvent,
MemberEmailChangeEvent,
MemberCreatedEvent,
- SubscriptionCreatedEvent,
MemberLinkClickEvent,
EmailSpamComplaintEvent,
Offer,
@@ -119,7 +118,6 @@ module.exports = function MembersAPI({
MemberStatusEvent,
MemberLoginEvent,
MemberCreatedEvent,
- SubscriptionCreatedEvent,
MemberLinkClickEvent,
MemberFeedback,
EmailSpamComplaintEvent,
@@ -202,7 +200,6 @@ module.exports = function MembersAPI({
paymentsService,
tiersService,
memberRepository,
- StripePrice,
allowSelfSignup,
magicLinkService,
stripeAPIService,
diff --git a/ghost/core/core/server/services/members/members-api/repositories/event-repository.js b/ghost/core/core/server/services/members/members-api/repositories/event-repository.js
index 513c67a7789..5a2ab78ec27 100644
--- a/ghost/core/core/server/services/members/members-api/repositories/event-repository.js
+++ b/ghost/core/core/server/services/members/members-api/repositories/event-repository.js
@@ -34,13 +34,11 @@ module.exports = class EventRepository {
MemberStatusEvent,
MemberLoginEvent,
MemberCreatedEvent,
- SubscriptionCreatedEvent,
MemberPaidSubscriptionEvent,
MemberLinkClickEvent,
MemberFeedback,
EmailSpamComplaintEvent,
Comment,
- labsService,
memberAttributionService,
urlService,
MemberEmailChangeEvent,
@@ -55,10 +53,8 @@ module.exports = class EventRepository {
this._MemberLoginEvent = MemberLoginEvent;
this._EmailRecipient = EmailRecipient;
this._Comment = Comment;
- this._labsService = labsService;
this._urlService = urlService;
this._MemberCreatedEvent = MemberCreatedEvent;
- this._SubscriptionCreatedEvent = SubscriptionCreatedEvent;
this._MemberLinkClickEvent = MemberLinkClickEvent;
this._MemberFeedback = MemberFeedback;
this._EmailSpamComplaintEvent = EmailSpamComplaintEvent;
diff --git a/ghost/core/core/server/services/members/service.js b/ghost/core/core/server/services/members/service.js
index 52a64300929..0262d45e4f4 100644
--- a/ghost/core/core/server/services/members/service.js
+++ b/ghost/core/core/server/services/members/service.js
@@ -7,7 +7,6 @@ const MembersConfigProvider = require('./members-config-provider');
const { makeImporter, makeExporter } = require('./import-export');
const { resolveInlineThreshold } = require('./import-export/config');
const MembersStats = require('./stats/members-stats');
-const memberJobs = require('./jobs');
const logging = require('@tryghost/logging');
const urlUtils = require('../../../shared/url-utils').default;
const settingsCache = require('../../../shared/settings-cache');
@@ -29,7 +28,7 @@ const messages = {
sslRequiredForStripe:
'Cannot run Ghost without SSL when Stripe is connected. Please update your url config to use "https://".',
remoteWebhooksInDevelopment:
- 'Cannot use remote webhooks in development. See https://ghost.org/docs/webhooks/#stripe-webhooks for developing with Stripe.',
+ 'Cannot use remote webhooks in development. See https://docs.ghost.org/webhooks/#stripe-webhooks for developing with Stripe.',
};
const ghostMailer = new GhostMailer();
@@ -216,9 +215,6 @@ module.exports = {
);
}
}
-
- // Schedule daily cron job to clean expired comp subs
- memberJobs.scheduleExpiredCompCleanupJob();
},
contentGating: require('./content-gating'),
diff --git a/ghost/core/core/server/services/route-settings/validation-errors.ts b/ghost/core/core/server/services/route-settings/validation-errors.ts
index 825bf4a6cf4..d09e5717e2d 100644
--- a/ghost/core/core/server/services/route-settings/validation-errors.ts
+++ b/ghost/core/core/server/services/route-settings/validation-errors.ts
@@ -6,7 +6,7 @@ const messages = {
validationError: 'The following definition "{at}" is invalid: {reason}',
};
-const DOCS_URL = 'https://ghost.org/docs/themes/routing/';
+const DOCS_URL = 'https://docs.ghost.org/themes/routing/';
const MAX_SHOWN_LENGTH = 40;
export type PathSegment = string | number;
diff --git a/ghost/core/core/server/services/route-settings/yaml-parser.js b/ghost/core/core/server/services/route-settings/yaml-parser.js
index 8a69f44b958..3a31baf2c5c 100644
--- a/ghost/core/core/server/services/route-settings/yaml-parser.js
+++ b/ghost/core/core/server/services/route-settings/yaml-parser.js
@@ -9,7 +9,7 @@ const messages = {
},
invalidYamlFormat: {
message: 'YAML input cannot be a plain string. Check the format of your YAML file.',
- help: 'https://ghost.org/docs/themes/routing/',
+ help: 'https://docs.ghost.org/themes/routing/',
},
};
diff --git a/ghost/core/core/server/services/settings-helpers/settings-helpers.js b/ghost/core/core/server/services/settings-helpers/settings-helpers.js
index 4f29f2a4b7e..e42ddfebaec 100644
--- a/ghost/core/core/server/services/settings-helpers/settings-helpers.js
+++ b/ghost/core/core/server/services/settings-helpers/settings-helpers.js
@@ -1,7 +1,7 @@
const net = require('net');
const tpl = require('@tryghost/tpl');
const errors = require('@tryghost/errors');
-const EmailAddressParser = require('../email-address/email-address-parser');
+const emailAddressParser = require('../email-address/email-address-parser');
const logging = require('@tryghost/logging');
const crypto = require('crypto');
const debug = require('@tryghost/debug')('services:settings-helpers');
@@ -130,11 +130,9 @@ class SettingsHelpers {
if (!supportAddress) {
// In the new flow, we make a difference between an empty setting (= use default) and a 'noreply' setting (=use noreply @ domain)
// Also keep the name of the default email!
- return EmailAddressParser.stringify(this.getDefaultEmail());
+ return emailAddressParser.stringify(this.getDefaultEmail());
}
- supportAddress = supportAddress || 'noreply';
-
// Any fromAddress without domain uses site domain, like default setting `noreply`
if (supportAddress.indexOf('@') < 0) {
return `${supportAddress}@${this.getDefaultEmailDomain()}`;
@@ -143,7 +141,7 @@ class SettingsHelpers {
}
/**
- * @deprecated Use getDefaultEmail().address (without name) or EmailAddressParser.stringify(this.getDefaultEmail()) (with name) instead
+ * @deprecated Use getDefaultEmail().address (without name) or emailAddressParser.stringify(this.getDefaultEmail()) (with name) instead
*/
getNoReplyAddress() {
return this.getDefaultEmailAddress();
@@ -157,7 +155,7 @@ class SettingsHelpers {
// parse the email here and remove the sender name
// E.g. when set to "bar"
const configAddress = this.config.get('mail:from');
- const parsed = EmailAddressParser.parse(configAddress);
+ const parsed = emailAddressParser.parse(configAddress);
if (parsed) {
return parsed;
}
diff --git a/ghost/core/core/server/services/slack-ping/slack-ping-service.ts b/ghost/core/core/server/services/slack-ping/slack-ping-service.ts
index edbeb1c91fa..8ab92b646f5 100644
--- a/ghost/core/core/server/services/slack-ping/slack-ping-service.ts
+++ b/ghost/core/core/server/services/slack-ping/slack-ping-service.ts
@@ -261,7 +261,7 @@ export class SlackPingService {
new errors.InternalServerError({
err: err,
context: tpl(messages.requestFailedError, { service: 'slack' }),
- help: tpl(messages.requestFailedHelp, { url: 'https://ghost.org/docs/' }),
+ help: tpl(messages.requestFailedHelp, { url: 'https://docs.ghost.org/' }),
}),
);
});
diff --git a/ghost/core/core/server/services/staff/staff-service-emails.js b/ghost/core/core/server/services/staff/staff-service-emails.js
index fe8ec7d224f..37558cb132c 100644
--- a/ghost/core/core/server/services/staff/staff-service-emails.js
+++ b/ghost/core/core/server/services/staff/staff-service-emails.js
@@ -2,7 +2,7 @@ const { promises: fs, readFileSync } = require('fs');
const path = require('path');
const moment = require('moment');
const { globSync } = require('glob');
-const EmailAddressParser = require('../email-address/email-address-parser');
+const emailAddressParser = require('../email-address/email-address-parser');
class StaffServiceEmails {
constructor({
@@ -593,7 +593,7 @@ class StaffServiceEmails {
}
get fromEmailAddress() {
- return EmailAddressParser.stringify(this.settingsHelpers.getDefaultEmail());
+ return emailAddressParser.stringify(this.settingsHelpers.getDefaultEmail());
}
extractInitials(name = '') {
diff --git a/ghost/core/core/server/services/stripe/config.js b/ghost/core/core/server/services/stripe/config.js
index 9a81fbee635..82c0df595c8 100644
--- a/ghost/core/core/server/services/stripe/config.js
+++ b/ghost/core/core/server/services/stripe/config.js
@@ -4,7 +4,7 @@ const labs = require('../../../shared/labs');
const messages = {
remoteWebhooksInDevelopment:
- 'Cannot use remote webhooks in development. See https://ghost.org/docs/webhooks/#stripe-webhooks for developing with Stripe.',
+ 'Cannot use remote webhooks in development. See https://docs.ghost.org/webhooks/#stripe-webhooks for developing with Stripe.',
};
// @TODO Refactor to a class w/ constructor
diff --git a/ghost/core/core/server/services/update-check/update-check-service.js b/ghost/core/core/server/services/update-check/update-check-service.js
index 29d8d034627..003b19ff5a7 100644
--- a/ghost/core/core/server/services/update-check/update-check-service.js
+++ b/ghost/core/core/server/services/update-check/update-check-service.js
@@ -99,7 +99,7 @@ class UpdateCheckService {
);
err.context = tpl(messages.checkingForUpdatesFailedError);
- err.help = tpl(messages.checkingForUpdatesFailedHelp, { url: 'https://ghost.org/docs/' });
+ err.help = tpl(messages.checkingForUpdatesFailedHelp, { url: 'https://docs.ghost.org/' });
this.logging.error(
{
@@ -120,7 +120,7 @@ class UpdateCheckService {
* Only the Ghost version is sent so the service can return the relevant
* release and security notifications. No site data is collected.
*
- * @see https://ghost.org/docs/concepts/config/#privacy
+ * @see https://docs.ghost.org/concepts/config/#privacy
* @returns {Promise}
*/
async updateCheckRequest() {
diff --git a/ghost/core/core/server/web/admin/controller.js b/ghost/core/core/server/web/admin/controller.js
index d7a586971fb..5a397871244 100644
--- a/ghost/core/core/server/web/admin/controller.js
+++ b/ghost/core/core/server/web/admin/controller.js
@@ -46,7 +46,7 @@ module.exports = function adminController(req, res) {
throw new errors.IncorrectUsageError({
message: tpl(messages.templateError.message, { templatePath }),
context: tpl(messages.templateError.context),
- help: tpl(messages.templateError.help, { link: 'https://ghost.org/docs/install/source/' }),
+ help: tpl(messages.templateError.help, { link: 'https://docs.ghost.org/install/source/' }),
err,
});
}
diff --git a/ghost/core/package.json b/ghost/core/package.json
index 7d71117bdc2..142330480dd 100644
--- a/ghost/core/package.json
+++ b/ghost/core/package.json
@@ -139,7 +139,7 @@
"@tryghost/string": "catalog:",
"@tryghost/tpl": "catalog:",
"@tryghost/url-utils": "5.2.6",
- "@tryghost/validator": "0.2.22",
+ "@tryghost/validator": "catalog:",
"@tryghost/version": "2.3.2",
"@tryghost/zip": "3.5.1",
"@x402/core": "catalog:",
diff --git a/ghost/core/test/e2e-api/admin/__snapshots__/posts.test.js.snap b/ghost/core/test/e2e-api/admin/__snapshots__/posts.test.js.snap
index 24a08745f1b..946fb37039b 100644
--- a/ghost/core/test/e2e-api/admin/__snapshots__/posts.test.js.snap
+++ b/ghost/core/test/e2e-api/admin/__snapshots__/posts.test.js.snap
@@ -1640,7 +1640,7 @@ Object {
"context": "Posts can have either a mobiledoc or a lexical property, never both. Both the mobiledoc and lexical properties are set, one must be null",
"details": null,
"ghostErrorCode": null,
- "help": "https://ghost.org/docs/admin-api/#the-post-object",
+ "help": "https://docs.ghost.org/admin-api/#the-post-object",
"id": StringMatching /\\[a-f0-9\\]\\{8\\}-\\[a-f0-9\\]\\{4\\}-\\[a-f0-9\\]\\{4\\}-\\[a-f0-9\\]\\{4\\}-\\[a-f0-9\\]\\{12\\}/,
"message": "Validation error, cannot save post.",
"property": "lexical",
@@ -1689,7 +1689,7 @@ Object {
"context": StringMatching /Invalid lexical structure\\\\\\.\\.\\*/,
"details": null,
"ghostErrorCode": null,
- "help": "https://ghost.org/docs/publishing/",
+ "help": "https://docs.ghost.org/publishing/",
"id": StringMatching /\\[a-f0-9\\]\\{8\\}-\\[a-f0-9\\]\\{4\\}-\\[a-f0-9\\]\\{4\\}-\\[a-f0-9\\]\\{4\\}-\\[a-f0-9\\]\\{12\\}/,
"message": "Validation error, cannot save post.",
"property": "lexical",
diff --git a/ghost/core/test/e2e-api/admin/posts-importer.test.js b/ghost/core/test/e2e-api/admin/posts-importer.test.js
index d79d900aebb..76fc0bc5a6e 100644
--- a/ghost/core/test/e2e-api/admin/posts-importer.test.js
+++ b/ghost/core/test/e2e-api/admin/posts-importer.test.js
@@ -12,6 +12,7 @@ const {
const { cacheInvalidateHeaderNotSet } = assertions;
const path = require('path');
const nock = require('nock');
+const papaparse = require('papaparse');
const models = require('../../../core/server/models');
const jobsService = require('../../../core/server/services/jobs');
const mediaInlinerService = require('../../../core/server/services/media-inliner');
@@ -102,6 +103,7 @@ describe('Posts Importer API', function () {
// Each test logs in as a different role โ reset the login rate limiter
// so the repeated logins don't trip spam prevention
await resetRateLimits();
+ mockManager.mockMail();
remoteImportedMediaUrls = [];
await Promise.all(getImportedAssetPaths().map((filePath) => fs.rm(filePath, { force: true })));
});
@@ -127,6 +129,190 @@ describe('Posts Importer API', function () {
.expect(cacheInvalidateHeaderNotSet());
});
+ it('emails the requesting user when an accepted CSV import finishes', async function () {
+ await agent.loginAsOwner();
+ const completionCsvPath = await csvFile(
+ 'posts-import-completion-email.csv',
+ 'title,html,status\n' +
+ 'Completion email created,A snippet from a post template A snippet from a post template A snippet from a post template A snippet from a post template A snippet from a post template A snippet from a post template A snippet from a post template A snippet from a post template A snippet from a post template A snippet from a post template A snippet from a post template A snippet from a post template A snippet from a post template A snippet from a post template ,
+ { timeoutMs = 5000, intervalMs = 25 } = {},
+): Promise {
+ const deadline = Date.now() + timeoutMs;
+ while (Date.now() < deadline) {
+ if (await check()) {
+ return true;
+ }
+ await new Promise((resolve) => {
+ setTimeout(resolve, intervalMs);
+ });
+ }
+ return false;
+}
+
+// Datetimes are inserted in the cross-dialect 'YYYY-MM-DD HH:mm:ss' format the
+// models use, so the job's UTC-day boundary comparison behaves as in production.
+function daysFromNow(days: number): string {
+ return moment.utc().add(days, 'days').format('YYYY-MM-DD HH:mm:ss');
+}
+
+describe('Job: Clean expired comped', function () {
+ let paidTierId: string;
+ const memberIds: string[] = [];
+
+ async function createCompedMember(email: string, expiryAt: string): Promise {
+ const member = await models.Member.add({
+ email,
+ name: email,
+ status: 'comped',
+ email_disabled: false,
+ });
+ memberIds.push(member.id);
+
+ await db.knex('members_products').insert({
+ id: ObjectId().toHexString(),
+ member_id: member.id,
+ product_id: paidTierId,
+ sort_order: 0,
+ expiry_at: expiryAt,
+ });
+
+ return member.id;
+ }
+
+ async function compedToFreeEvents(memberId: string): Promise {
+ return db
+ .knex('members_status_events')
+ .where({ member_id: memberId, from_status: 'comped', to_status: 'free' })
+ .select('*');
+ }
+
+ beforeAll(async function () {
+ await agentProvider.getAdminAPIAgent();
+ await fixtureManager.init('newsletters', 'members:newsletters');
+
+ const paidTier = await models.Product.findOne({ type: 'paid' }, { require: true });
+ paidTierId = paidTier.id;
+ });
+
+ afterAll(async function () {
+ await db.knex('members_products').whereIn('member_id', memberIds).del();
+ await db.knex('members_status_events').whereIn('member_id', memberIds).del();
+ await db.knex('members').whereIn('id', memberIds).del();
+ });
+
+ it('cleans up expired comped members exactly once and leaves the rest alone', async function () {
+ const expiredMemberId = await createCompedMember('expired-comped@example.com', daysFromNow(-2));
+ const activeMemberId = await createCompedMember('active-comped@example.com', daysFromNow(2));
+ // Eligibility is strictly before the start of the current UTC day, so an
+ // expiry later today must survive - pins the cutoff's cross-dialect
+ // format (an ISO cutoff string wrongly matches this row on SQLite)
+ const sameDayMemberId = await createCompedMember(
+ 'same-day-comped@example.com',
+ moment.utc().endOf('day').format('YYYY-MM-DD HH:mm:ss'),
+ );
+
+ const editedEvents: any[] = [];
+ const onMemberEdited = (model: any) => {
+ editedEvents.push(model);
+ };
+ events.on('member.edited', onMemberEdited);
+
+ // Installed before the first dispatch and consulted by count, so the
+ // repeat-run wait below cannot be satisfied by the first run's straggler
+ // completion log (the task emits events before it logs completion)
+ const loggingInfoSpy = sinon.spy(logging, 'info');
+ const completedLogCount = () =>
+ loggingInfoSpy
+ .getCalls()
+ .filter((call) => call.args[0]?.system?.event === 'clean_expired_comped.completed').length;
+
+ try {
+ await getJobsService().dispatch(new CleanExpiredCompedJob());
+
+ const cleaned = await waitFor(async () => {
+ const member = await models.Member.findOne({ id: expiredMemberId });
+ return member.get('status') === 'free';
+ });
+ assert.ok(cleaned, 'The dispatched job moves the expired comped member to free');
+
+ const expiredRelations = await db
+ .knex('members_products')
+ .where({ member_id: expiredMemberId });
+ assert.equal(expiredRelations.length, 0, 'The expired product relation is deleted');
+
+ const statusEvents = await compedToFreeEvents(expiredMemberId);
+ assert.equal(statusEvents.length, 1, 'Exactly one comped->free status event is recorded');
+
+ // The model event is emitted after the database work, so wait for it too
+ const eventEmitted = await waitFor(async () => {
+ return editedEvents.some((model) => model.id === expiredMemberId);
+ });
+ assert.ok(eventEmitted, 'A member.edited model event is emitted for the changed member');
+ const emittedModel = editedEvents.find((model) => model.id === expiredMemberId);
+ assert.equal(emittedModel._previousAttributes.status, 'comped');
+ assert.equal(emittedModel._changed.status, 'free');
+
+ const activeMember = await models.Member.findOne({ id: activeMemberId });
+ assert.equal(activeMember.get('status'), 'comped', 'A future expiry is left alone');
+ const activeRelations = await db
+ .knex('members_products')
+ .where({ member_id: activeMemberId });
+ assert.equal(activeRelations.length, 1, 'The unexpired product relation is kept');
+
+ const sameDayMember = await models.Member.findOne({ id: sameDayMemberId });
+ assert.equal(
+ sameDayMember.get('status'),
+ 'comped',
+ 'An expiry later on the current UTC day is not yet eligible',
+ );
+ const sameDayRelations = await db
+ .knex('members_products')
+ .where({ member_id: sameDayMemberId });
+ assert.equal(sameDayRelations.length, 1, 'The same-day product relation is kept');
+
+ const firstRunLogged = await waitFor(async () => completedLogCount() === 1);
+ assert.ok(firstRunLogged, 'The first run logs its completion event');
+
+ // A repeat run must be a no-op: no duplicate status history, no new events
+ editedEvents.length = 0;
+ await getJobsService().dispatch(new CleanExpiredCompedJob());
+ const repeatCompleted = await waitFor(async () => completedLogCount() === 2);
+ assert.ok(repeatCompleted, 'The repeat run finishes and logs its completion event');
+
+ const statusEventsAfterRepeat = await compedToFreeEvents(expiredMemberId);
+ assert.equal(statusEventsAfterRepeat.length, 1, 'A repeat run adds no status history');
+ assert.equal(
+ editedEvents.filter((model) => model.id === expiredMemberId).length,
+ 0,
+ 'A repeat run emits no further model events',
+ );
+ const activeMemberAfterRepeat = await models.Member.findOne({ id: activeMemberId });
+ assert.equal(activeMemberAfterRepeat.get('status'), 'comped');
+ } finally {
+ loggingInfoSpy.restore();
+ events.removeListener('member.edited', onMemberEdited);
+ }
+ });
+
+ it('does not change a member who is no longer comped when their relation expires', async function () {
+ const memberId = await createCompedMember('paid-since-comped@example.com', daysFromNow(-2));
+ await db.knex('members').where({ id: memberId }).update({ status: 'paid' });
+
+ await getJobsService().dispatch(new CleanExpiredCompedJob());
+
+ const relationDeleted = await waitFor(async () => {
+ const relations = await db.knex('members_products').where({ member_id: memberId });
+ return relations.length === 0;
+ });
+ assert.ok(relationDeleted, 'The expired product relation is still deleted');
+
+ const member = await models.Member.findOne({ id: memberId });
+ assert.equal(member.get('status'), 'paid', 'A member who is not comped keeps their status');
+
+ const statusEvents = await compedToFreeEvents(memberId);
+ assert.equal(statusEvents.length, 0, 'No status event is recorded for a non-comped member');
+ });
+
+ // The migration's core safety claim is that the delete, member update and
+ // status-event insert commit or roll back together. This exercises a real
+ // database transaction that fails on its last write and proves nothing
+ // else stuck.
+ it('rolls back every write when the transaction fails part-way', async function () {
+ const cleanExpiredComped =
+ require('../../../../core/server/services/members/jobs/clean-expired-comped-task').default;
+ const memberId = await createCompedMember('rollback-comped@example.com', daysFromNow(-2));
+
+ const realKnex = db.knex;
+ const failingDb = {
+ knex: {
+ transaction: (callback: (trx: unknown) => unknown) =>
+ realKnex.transaction((trx: unknown) => {
+ const proxied = new Proxy(trx as object, {
+ apply(target, thisArg, args: unknown[]) {
+ if (args[0] === 'members_status_events') {
+ return {
+ insert: () => Promise.reject(new Error('forced mid-transaction failure')),
+ };
+ }
+ return Reflect.apply(target as (...a: unknown[]) => unknown, thisArg, args);
+ },
+ });
+ return callback(proxied);
+ }),
+ },
+ };
+
+ const stubLogging = { info: sinon.stub(), warn: sinon.stub(), error: sinon.stub() };
+ const stubSentry = { captureException: sinon.stub() };
+ const editedEvents: unknown[] = [];
+ const onMemberEdited = (model: unknown) => {
+ editedEvents.push(model);
+ };
+ events.on('member.edited', onMemberEdited);
+
+ try {
+ await assert.rejects(
+ () =>
+ cleanExpiredComped({
+ db: failingDb,
+ models,
+ events,
+ logging: stubLogging,
+ sentry: stubSentry,
+ }),
+ /forced mid-transaction failure/,
+ );
+
+ const relations = await db.knex('members_products').where({ member_id: memberId });
+ assert.equal(relations.length, 1, 'The expired product relation survives the rollback');
+
+ const member = await models.Member.findOne({ id: memberId });
+ assert.equal(member.get('status'), 'comped', 'The member keeps their comped status');
+
+ const statusEvents = await compedToFreeEvents(memberId);
+ assert.equal(statusEvents.length, 0, 'No status history sticks');
+
+ assert.equal(editedEvents.length, 0, 'No model event is emitted for rolled-back work');
+ } finally {
+ events.removeListener('member.edited', onMemberEdited);
+ }
+ });
+});
diff --git a/ghost/core/test/integration/services/members/send-gift-reminders.test.js b/ghost/core/test/integration/services/members/send-gift-reminders.test.js
index e07b0fa587c..1f9677cfeaa 100644
--- a/ghost/core/test/integration/services/members/send-gift-reminders.test.js
+++ b/ghost/core/test/integration/services/members/send-gift-reminders.test.js
@@ -25,7 +25,7 @@ describe('Gift reminder processing', function () {
await agent.loginAsOwner();
giftService = require('../../../../core/server/services/gifts');
- await giftService.init();
+ giftService.init();
paidTier = await models.Product.findOne({ type: 'paid' }, { require: true });
});
diff --git a/ghost/core/test/unit/frontend/apps/private-blogging/controller.test.js b/ghost/core/test/unit/frontend/apps/private-blogging/controller.test.js
index 9080498f6b3..13fc836f979 100644
--- a/ghost/core/test/unit/frontend/apps/private-blogging/controller.test.js
+++ b/ghost/core/test/unit/frontend/apps/private-blogging/controller.test.js
@@ -256,6 +256,32 @@ describe('private.hbs template translation', function () {
assert(html.includes('src="/private.js"'));
});
+ it('renders the access-code form instead of the signup form for an authenticated member', function () {
+ const context = {
+ member: { email: 'member@example.com' },
+ site: {
+ title: 'Test',
+ url: 'http://test.local',
+ locale: 'en',
+ allow_self_signup: true,
+ admin_url: 'http://test.local/ghost/',
+ },
+ };
+ const html = renderPrivateTemplate(context);
+
+ assertExists(html);
+ assert(!html.includes('data-ghost-private-subscribe-form'));
+ assert.equal(
+ (html.match(/
Including ${formatNumber(remaining)} more.
` + : ''; + const tag = encodeURIComponent(importTagSlug(run.id)); + const postsUrl = escapeHTML(new URL(`#/posts?tag=${tag}`, adminUrl).href); + const pagesUrl = escapeHTML(new URL(`#/pages?tag=${tag}`, adminUrl).href); + let heading = 'Imported posts'; + if (hasPages) { + heading = hasPosts ? 'Imported posts and pages' : 'Imported pages'; + } + const filteredLinks = [ + hasPosts ? `View imported posts` : '', + hasPages ? `View imported pages` : '', + ] + .filter(Boolean) + .join(' ยท '); + const preview = links + ? `- ${links}
${heading}
+ ${preview} + ${remainingCopy} +${filteredLinks}
`; +} + +function renderCompletionEmail(run: ImportRun, recipient: string, adminUrl: string): string { + const counts = countsFor(run); + const heading = headingFor(run, counts); + const warningCopy = counts.warningRows + ? `${formatNumber(counts.warningRows)} ${counts.warningRows === 1 ? 'post has' : 'posts have'} warnings.
` + : ''; + const failureCopy = + run.status === 'failed' + ? 'Something went wrong on our end before the import could finish. There is nothing wrong with your file, and it is safe to try again.
' + : ''; + const escapedRecipient = escapeHTML(recipient); + + return ` + + + + +
+ 
+
+
+`;
+}
+
+export default function buildCompletionEmail(
+ run: ImportRun,
+ recipient: string,
+ adminUrl: string,
+): CompletionEmailPayload {
+ const counts = countsFor(run);
+ const report = buildImportReport(run);
+ const errorsFile = buildErrorsFile(run);
+ const attachments: CompletionEmailPayload['attachments'] = [];
+ if (report) {
+ attachments.push({
+ filename: 'report.csv',
+ content: report,
+ contentType: 'text/csv',
+ contentDisposition: 'attachment',
+ });
+ }
+ if (errorsFile) {
+ attachments.push({
+ filename: 'errors.csv',
+ content: errorsFile,
+ contentType: 'text/csv',
+ contentDisposition: 'attachment',
+ });
+ }
+ return {
+ to: recipient,
+ subject: headingFor(run, counts),
+ html: renderCompletionEmail(run, recipient, adminUrl),
+ forceTextContent: true,
+ attachments,
+ };
+}
diff --git a/ghost/core/core/server/services/content-import/import/errors-file.ts b/ghost/core/core/server/services/content-import/import/errors-file.ts
new file mode 100644
index 00000000000..7d727d96b3e
--- /dev/null
+++ b/ghost/core/core/server/services/content-import/import/errors-file.ts
@@ -0,0 +1,47 @@
+import { serialize } from '../csv';
+import type { ImportRun, RowOutcome } from './store';
+
+const ANNOTATION_NAMES = ['import_status', 'import_reason', 'import_media_failures'] as const;
+
+function isActionable(row: RowOutcome): boolean {
+ return Boolean(row.source) && row.status === 'failed';
+}
+
+function uniqueColumnName(preferred: string, used: Set
+ ${heading}
+ ${failureCopy} +The import processed ${formatNumber(run.total)} ${run.total === 1 ? 'row' : 'rows'}:
+- ${summaryItems(counts)}
This email was sent to ${escapedRecipient}.
+Created
,published\n' + + 'Completion email draft,Draft
,draft\n', + ); + + const { body } = await agent + .post('posts/upload/') + .attach('postsfile', completionCsvPath) + .expectStatus(202); + await jobsService.allSettled(); + + const email = mockManager.assert.sentEmail({ + subject: 'Your content import is complete', + to: 'jbloggs@example.com', + }); + assert.match(email.html, /processed 2 rows/); + assert.match(email.html, /Created:<\/strong> 2/); + assert.match(email.html, /Updated:<\/strong> 0/); + assert.match(email.html, /Skipped:<\/strong> 0/); + assert.match(email.html, /Failed:<\/strong> 0/); + + const draft = await models.Post.findOne({ title: 'Completion email draft', status: 'all' }); + assert.ok(draft); + assert.match(email.html, /Completion email created/); + assert.match(email.html, /\/completion-email-created\//); + assert.match(email.html, /Imported posts/); + assert.doesNotMatch(email.html, /Imported posts and pages/); + assert.ok( + email.html.includes(`/#/editor/post/${draft.id}`), + 'the draft links to its Admin editor', + ); + assert.match(email.html, new RegExp(`/#/posts\\?tag=hash-import-run-${body.meta.import_id}`)); + assert.doesNotMatch(email.html, /View imported pages/); + assert.doesNotMatch( + email.html, + new RegExp(`/#/pages\\?tag=hash-import-run-${body.meta.import_id}`), + ); + }); + + it('attaches a report that classifies same-run and pre-existing duplicates', async function () { + await agent.loginAsOwner(); + const preExistingPath = await csvFile( + 'posts-import-report-pre-existing.csv', + 'title,slug\nPre-existing report post,report-pre-existing\n', + ); + + await agent.post('posts/upload/').attach('postsfile', preExistingPath).expectStatus(202); + await jobsService.allSettled(); + mockManager.assert.sentEmail({ subject: 'Your content import is complete' }); + + const reportPath = await csvFile( + 'posts-import-report.csv', + [ + 'title,slug,status,custom_excerpt,authors,author_emails', + 'First in this run,report-same-run,draft,,,', + 'Duplicate in this run,report-same-run,draft,,,', + 'Pre-existing duplicate,report-pre-existing,draft,,,', + ',,,,,', + 'Invalid status,report-invalid-status,scheduled,,,', + `Write failure,report-write-failure,draft,${'x'.repeat(301)},,`, + 'Warning success,report-warning,draft,,Warning Author,not-an-email', + ',,,,,', + ' , , , , , ', + ].join('\n'), + ); + + await agent.post('posts/upload/').attach('postsfile', reportPath).expectStatus(202); + await jobsService.allSettled(); + + const email = mockManager.assert.sentEmail({ subject: 'Your content import is complete' }); + assert.match(email.html, /processed 6 rows/); + assert.match(email.html, /Created:<\/strong> 2/); + assert.match(email.html, /Updated:<\/strong> 0/); + assert.match(email.html, /Skipped:<\/strong> 2/); + assert.match(email.html, /Failed:<\/strong> 2/); + assert.equal(email.attachments.length, 2); + const report = email.attachments.find(({ filename }) => filename === 'report.csv'); + assert.ok(report); + assert.equal(report.contentType, 'text/csv'); + + const { data: rows } = papaparse.parse(report.content.trim(), { + header: true, + }); + assert.equal(rows.length, 6); + const sameRun = rows.find((row) => row.title === 'Duplicate in this run'); + assert.equal(sameRun.outcome, 'duplicate'); + assert.equal(sameRun.duplicate_origin, 'this_import'); + assert.equal(sameRun.matched_by, 'slug'); + const preExisting = rows.find((row) => row.title === 'Pre-existing duplicate'); + assert.equal(preExisting.outcome, 'duplicate'); + assert.equal(preExisting.duplicate_origin, 'pre_existing'); + assert.equal(preExisting.matched_by, 'slug'); + assert.equal(rows.find((row) => row.title === 'Invalid status').outcome, 'failed'); + assert.equal(rows.find((row) => row.title === 'Invalid status').line, '6'); + assert.equal(rows.find((row) => row.title === 'Write failure').outcome, 'failed'); + assert.equal(rows.find((row) => row.title === 'Write failure').line, '7'); + assert.match(rows.find((row) => row.title === 'Warning success').warnings, /assigned Owner/); + assert.equal(rows.find((row) => row.title === 'First in this run').outcome, 'created'); + + const errorsFile = email.attachments.find(({ filename }) => filename === 'errors.csv'); + assert.ok(errorsFile); + const { data: errorRows, meta } = papaparse.parse(errorsFile.content.trim(), { header: true }); + assert.deepEqual(meta.fields.slice(0, 7), [ + 'import_status', + 'title', + 'slug', + 'status', + 'custom_excerpt', + 'authors', + 'author_emails', + ]); + assert.deepEqual( + errorRows.map(({ title }) => title), + ['Invalid status', 'Write failure'], + ); + assert.equal( + errorRows.some(({ title }) => title === 'Duplicate in this run'), + false, + ); + assert.equal( + errorRows.some(({ title }) => title === 'Warning success'), + false, + ); + }); + + it('preserves mapped ZIP source columns and avoids annotation collisions', async function () { + await agent.loginAsOwner(); + const zipPath = await zipFile('posts-import-errors-source.zip', { + 'wrapper/posts.csv': + 'Body,Headline,State,import_status\nKeep source cells
,ZIP invalid,scheduled,publisher value\n', + }); + const form = new FormData(); + form.append('mapping[Body]', 'html'); + form.append('mapping[Headline]', 'title'); + form.append('mapping[State]', 'status'); + form.append('mapping[import_status]', ''); + form.append('postsfile', await fs.readFile(zipPath), { + filename: path.basename(zipPath), + contentType: 'application/zip', + }); + + await agent.post('posts/upload/').body(form).expectStatus(202); + await jobsService.allSettled(); + + const email = mockManager.assert.sentEmail({ + subject: 'Your content import was unsuccessful', + }); + assert.match(email.html, /Skipped:<\/strong> 0/); + assert.match(email.html, /Failed:<\/strong> 1/); + const errorsFile = email.attachments.find(({ filename }) => filename === 'errors.csv'); + assert.ok(errorsFile); + const parsed = papaparse.parse(errorsFile.content.trim(), { header: true }); + assert.deepEqual(parsed.meta.fields, [ + 'import_status_2', + 'Body', + 'Headline', + 'State', + 'import_status', + 'import_reason', + 'import_media_failures', + ]); + assert.equal(parsed.data[0].Body, 'Keep source cells
'); + assert.equal(parsed.data[0].Headline, 'ZIP invalid'); + assert.equal(parsed.data[0].State, 'scheduled'); + assert.equal(parsed.data[0].import_status, 'publisher value'); + assert.equal(parsed.data[0].import_status_2, 'failed'); + + const retryForm = new FormData(); + retryForm.append('mapping[Body]', 'html'); + retryForm.append('mapping[Headline]', 'title'); + retryForm.append('mapping[State]', 'status'); + retryForm.append('postsfile', Buffer.from(errorsFile.content), { + filename: 'errors.csv', + contentType: 'text/csv', + }); + await agent.post('posts/upload/').body(retryForm).expectStatus(202); + await jobsService.allSettled(); + mockManager.assert.sentEmail({ subject: 'Your content import was unsuccessful' }); + }); + it('Keeps content import initialization idempotent and rejects invalid service requests', async function () { const contentImportService = await import('../../../core/server/services/content-import/index.ts?coverage-lifecycle'); @@ -260,6 +446,19 @@ describe('Posts Importer API', function () { status: 'all', }); assert.ok(continuedPost); + + const email = mockManager.assert.sentEmail({ subject: 'Your content import is complete' }); + const errorsFile = email.attachments.find(({ filename }) => filename === 'errors.csv'); + assert.ok(errorsFile); + const parsed = papaparse.parse(errorsFile.content.trim(), { header: true }); + assert.equal(parsed.data.length, 1); + assert.equal(parsed.data[0].title, 'Unsupported remote media'); + assert.deepEqual(JSON.parse(parsed.data[0].import_media_failures), [ + { + sourceUrl: `${origin}/unsupported.exe`, + reason: 'No configured storage accepts this media file.', + }, + ]); }); it('preserves current-site media URLs without fetching or validating them', async function () { diff --git a/ghost/core/test/e2e-api/content/__snapshots__/posts.test.js.snap b/ghost/core/test/e2e-api/content/__snapshots__/posts.test.js.snap index fadfd9af882..ba2734d67d7 100644 --- a/ghost/core/test/e2e-api/content/__snapshots__/posts.test.js.snap +++ b/ghost/core/test/e2e-api/content/__snapshots__/posts.test.js.snap @@ -70,15 +70,15 @@ Object { <article class=\\"article {{post_class}}\\"> <h1>{{title}}</h1> - + {{#if feature_image}} <img src=\\"{{feature_image}}\\" alt=\\"Feature image\\" /> {{/if}} - + {{content}} </article> -{{/post}}See? Not that scary! But still completely optional.
If you're interested in creating your own Ghost theme, check out our extensive theme documentation for a full guide to all the different template variables and helpers which are available.
", +{{/post}}See? Not that scary! But still completely optional.
If you're interested in creating your own Ghost theme, check out our extensive theme documentation for a full guide to all the different template variables and helpers which are available.
", "id": "6194d3ce51e2700162531a76", "meta_description": null, "meta_title": null, @@ -519,15 +519,15 @@ Object { <article class=\\"article {{post_class}}\\"> <h1>{{title}}</h1> - + {{#if feature_image}} <img src=\\"{{feature_image}}\\" alt=\\"Feature image\\" /> {{/if}} - + {{content}} </article> -{{/post}}See? Not that scary! But still completely optional.
If you're interested in creating your own Ghost theme, check out our extensive theme documentation for a full guide to all the different template variables and helpers which are available.
", +{{/post}}See? Not that scary! But still completely optional.
If you're interested in creating your own Ghost theme, check out our extensive theme documentation for a full guide to all the different template variables and helpers which are available.
", "id": "6194d3ce51e2700162531a76", "meta_description": null, "meta_title": null, @@ -671,7 +671,7 @@ Object { {{content}} </article> -{{/post}}See? Not that scary! But still completely optional.
If you're interested in creating your own Ghost theme, check out our extensive theme documentation for a full guide to all the different template variables and helpers which are available.
", +{{/post}}See? Not that scary! But still completely optional.
If you're interested in creating your own Ghost theme, check out our extensive theme documentation for a full guide to all the different template variables and helpers which are available.
", "id": "6194d3ce51e2700162531a76", "meta_description": null, "meta_title": null, @@ -921,7 +921,7 @@ Object { {{content}} </article> -{{/post}}See? Not that scary! But still completely optional.
If you're interested in creating your own Ghost theme, check out our extensive theme documentation for a full guide to all the different template variables and helpers which are available.
", +{{/post}}See? Not that scary! But still completely optional.
If you're interested in creating your own Ghost theme, check out our extensive theme documentation for a full guide to all the different template variables and helpers which are available.
", "id": "6194d3ce51e2700162531a76", "meta_description": null, "meta_title": null, @@ -2761,7 +2761,7 @@ Object { {{content}} </article> -{{/post}}See? Not that scary! But still completely optional.
If you're interested in creating your own Ghost theme, check out our extensive theme documentation for a full guide to all the different template variables and helpers which are available.
", +{{/post}}See? Not that scary! But still completely optional.
If you're interested in creating your own Ghost theme, check out our extensive theme documentation for a full guide to all the different template variables and helpers which are available.
", "id": "6194d3ce51e2700162531a76", "meta_description": null, "meta_title": null, @@ -4270,7 +4270,7 @@ Object { {{content}} </article> -{{/post}}See? Not that scary! But still completely optional.
If you're interested in creating your own Ghost theme, check out our extensive theme documentation for a full guide to all the different template variables and helpers which are available.
", +{{/post}}See? Not that scary! But still completely optional.
If you're interested in creating your own Ghost theme, check out our extensive theme documentation for a full guide to all the different template variables and helpers which are available.
", "id": "6194d3ce51e2700162531a76", "meta_description": null, "meta_title": null, @@ -5135,7 +5135,7 @@ Object { {{content}} </article> -{{/post}}See? Not that scary! But still completely optional.
If you're interested in creating your own Ghost theme, check out our extensive theme documentation for a full guide to all the different template variables and helpers which are available.
", +{{/post}}See? Not that scary! But still completely optional.
If you're interested in creating your own Ghost theme, check out our extensive theme documentation for a full guide to all the different template variables and helpers which are available.
", "id": "6194d3ce51e2700162531a76", "meta_description": null, "meta_title": null, diff --git a/ghost/core/test/e2e-webhooks/posts-importer.test.ts b/ghost/core/test/e2e-webhooks/posts-importer.test.ts index 32cb70aca79..e709ad138db 100644 --- a/ghost/core/test/e2e-webhooks/posts-importer.test.ts +++ b/ghost/core/test/e2e-webhooks/posts-importer.test.ts @@ -133,8 +133,11 @@ describe('CSV content import side-effects', function () { // Zero webhooks: the receiver never recorded a request assert.equal(webhookMockReceiver.body, undefined, 'no webhook fired for the imported posts'); - // Zero transactional mail - mockManager.assert.sentEmailCount(0); + // The only mail side effects are the completion emails for the two + // accepted import runs. No per-post or newsletter mail is sent. + mockManager.assert.sentEmail({ subject: 'Your content import is complete' }); + mockManager.assert.sentEmail({ subject: 'Your content import is complete' }); + mockManager.assert.sentEmailCount(2); // Zero newsletters: a newsletter can only be attached via the API layer, which // the importer never touches. diff --git a/ghost/core/test/integration/services/members/clean-expired-comped.test.ts b/ghost/core/test/integration/services/members/clean-expired-comped.test.ts new file mode 100644 index 00000000000..5d93e971905 --- /dev/null +++ b/ghost/core/test/integration/services/members/clean-expired-comped.test.ts @@ -0,0 +1,258 @@ +import assert from 'node:assert/strict'; +import sinon from 'sinon'; +import CleanExpiredCompedJob from '../../../../core/server/services/members/jobs/clean-expired-comped-job'; + +const moment = require('moment'); +const logging = require('@tryghost/logging'); +const ObjectId = require('bson-objectid').default; +const { agentProvider, fixtureManager } = require('../../../utils/e2e-framework'); +const models = require('../../../../core/server/models'); +const db = require('../../../../core/server/data/db'); +const events = require('../../../../core/server/lib/common/events'); +const { getInstance: getJobsService } = require('../../../../core/server/services/jobs-service'); + +async function waitFor( + check: () => Promise