diff --git a/src/renderer/src/crud/loadData.ts b/src/renderer/src/crud/loadData.ts index c01b15d4b..c6b0b2360 100644 --- a/src/renderer/src/crud/loadData.ts +++ b/src/renderer/src/crud/loadData.ts @@ -171,14 +171,14 @@ export async function insertData( if (Array.isArray(rec)) rec = rec[0] as InitializedRecord; //won't be... rec.attributes = { ...item.attributes }; oparray.push(tb.updateRecord(rec).toOperation()); - if (rec.type === 'project') { - if (isProject) project = rec as ProjectD; + if (rec.type === 'project' && isProject) { + project = rec as ProjectD; await saveOfflineProject( rec as ProjectD, memory, backup, - isProject ? snapshotDate : undefined, - isImport && isProject + snapshotDate, + isImport ); } for (const rel in item.relationships) { @@ -206,14 +206,15 @@ export async function insertData( try { if (typeof item.id === 'number') item = rn.normalizeRecord(item); oparray.push(tb.addRecord(item).toOperation()); - if (item.type === 'project') { - if (isProject) project = item as ProjectD; + //don't mess with any offline project data for supporting projects + if (item.type === 'project' && isProject) { + project = item as ProjectD; await saveOfflineProject( item as ProjectD, memory, backup, - isProject ? snapshotDate : undefined, - isImport && isProject + snapshotDate, + isImport ); } } catch (errResult: unknown) { diff --git a/src/renderer/src/model/orgKeyterm.tsx b/src/renderer/src/model/orgKeyterm.tsx index d08407c87..4b56ce96f 100644 --- a/src/renderer/src/model/orgKeyterm.tsx +++ b/src/renderer/src/model/orgKeyterm.tsx @@ -6,7 +6,7 @@ export interface OrgKeyterm extends BaseModel { domain: string; definition: string; category: string; - offlineid: string; + offlineId: string; dateCreated: string; dateUpdated: string; lastModifiedBy: number; diff --git a/src/renderer/src/model/sectionResourceUser.ts b/src/renderer/src/model/sectionResourceUser.ts index 63699f6f0..921d8e517 100644 --- a/src/renderer/src/model/sectionResourceUser.ts +++ b/src/renderer/src/model/sectionResourceUser.ts @@ -6,6 +6,7 @@ export interface SectionResourceUser extends BaseModel { dateCreated: string; dateUpdated: string; lastModifiedBy: number; + offlineId: string; }; relationships?: { sectionresource: RecordRelationship; diff --git a/src/renderer/src/schema.tsx b/src/renderer/src/schema.tsx index 0da795a97..a71986347 100644 --- a/src/renderer/src/schema.tsx +++ b/src/renderer/src/schema.tsx @@ -744,6 +744,7 @@ if (requestedSchema > 3 && schemaDefinition.models) { dateCreated: { type: 'string' }, // datetime dateUpdated: { type: 'string' }, // datetime lastModifiedBy: { type: 'number' }, //bkwd compat only + offlineId: { type: 'string' }, }, relationships: { sectionresource: { diff --git a/src/renderer/src/store/importexport/electronExport.tsx b/src/renderer/src/store/importexport/electronExport.tsx index d0978cc33..bcd603b76 100644 --- a/src/renderer/src/store/importexport/electronExport.tsx +++ b/src/renderer/src/store/importexport/electronExport.tsx @@ -2,7 +2,6 @@ import { ExportType, FileResponse } from './types'; import path from 'path-browserify'; import { DateTime } from 'luxon'; import { - Project, Organization, Plan, OfflineProject, @@ -10,20 +9,7 @@ import { OrgWorkflowStep, ProjectD, UserD, - DiscussionD, - OrgKeytermD, - OrgKeytermTargetD, - IntellectualPropertyD, MediaFileD, - OrganizationD, - GroupD, - GroupMembershipD, - PlanD, - SectionD, - PassageD, - SectionResourceD, - ArtifactCategoryD, - SharedResourceD, } from '../../model'; import Memory from '@orbit/memory'; import { getSerializer } from '../../serializers/getSerializer'; @@ -36,9 +22,7 @@ import { getBurritoMeta, scriptureFullPath, IBurritoMeta, - IExportArtifacts, IExportScripturePath, - mediaArtifacts, fileInfo, updateableFiles, staticFiles, @@ -55,9 +39,9 @@ import { createPathFolder, } from '../../utils'; import IndexedDBSource from '@orbit/indexeddb'; -import { BaseModel, BaseModelD } from '../../model/baseModel'; import { backupToMemory } from '../../crud/syncToMemory'; import { MainAPI } from '@model/main-api'; +import { createExportCollector } from './exportTableRecs'; const ipc = window?.api as MainAPI; export async function electronExport( @@ -288,556 +272,19 @@ export async function electronExport( await ipc?.zipAddLocal(zip, fontfile, PathType.FONTS, items[i]); } }; - const GroupMemberships = (project: Project) => { - const groupid = related(project, 'group'); - return memory.cache.query((q) => - q.findRecords('groupmembership').filter({ - relation: 'group', - record: { type: 'group', id: groupid }, - }) - ) as GroupMembershipD[]; - }; - - const Plans = (project: ProjectD) => - memory.cache.query((q) => - q.findRecords('plan').filter({ - relation: 'project', - record: { type: 'project', id: project.id }, - }) - ) as PlanD[]; - - const Sections = (project: ProjectD) => { - //can't get this to work... - //let plans = Plans(project).map((pl) => { - // return { type: 'plan', id: pl.id }; - //}); - //let sections = memory.cache.query((q) => - // q.findRecords('section').filter({ relation: 'plan', records: plans }) - //) as Section[]; - const plans = Plans(project).map((pl) => pl.id); - const allsections = memory.cache.query((q) => - q.findRecords('section') - ) as SectionD[]; - return allsections.filter((s) => plans.includes(related(s, 'plan'))); - }; - - const Passages = (project: ProjectD) => { - const sections = Sections(project).map((s) => s.id); - const passages = memory.cache.query((q) => - q.findRecords('passage') - ) as PassageD[]; - return passages.filter((p) => sections.includes(related(p, 'section'))); - }; - const SectionResources = (project: ProjectD) => { - const sections = Sections(project).map((s) => s.id); - const sectionresources = ( - memory.cache.query((q) => - q.findRecords('sectionresource') - ) as SectionResourceD[] - ).filter((r) => sections.includes(related(r, 'section'))); - return sectionresources; - }; - - const HighestByPassage = (media: MediaFileD[]) => { - const highest: MediaFileD[] = []; - let psg = ''; - media - .sort((a, b) => - related(a, 'passage') === related(b, 'passage') - ? a.attributes.versionNumber > b.attributes.versionNumber - ? 1 - : -1 - : related(a, 'passage') > related(b, 'passage') - ? 1 - : -1 - ) - .forEach((m) => { - if (related(m, 'passage') !== psg) { - highest.push(m); - psg = related(m, 'passage'); - } - }); - return highest; - }; - const SourceMedia = (project: ProjectD) => { - const sectionresourcemedia = SectionResources(project).map( - (r) => related(r, 'mediafile') as string - ); - const media = memory.cache.query((q) => - q.findRecords('mediafile') - ) as MediaFileD[]; - - //get the mediafiles associated with section resources - const resourcemediafiles = media.filter((m) => - sectionresourcemedia.includes(m.id) - ); - - //now get any shared resource mediafiles associated with those mediafiles - const sourcemediafiles = media.filter( - (m) => - m.attributes?.readyToShare && - resourcemediafiles - .map((r) => related(r, 'resourcePassage')) - .includes(related(m, 'passage')) - ); - return HighestByPassage(sourcemediafiles); - }; - const sharedNotePassageIds = (project: ProjectD) => { - const psgs = Passages(project).filter( - (p) => related(p, 'sharedResource') !== null - ); - const sharednotesids = psgs.map( - (p) => related(p, 'sharedResource') as string - ); - const supportingNotes = ( - memory.cache.query((q) => - q.findRecords('sharedresource') - ) as SharedResourceD[] - ) - .filter((r) => sharednotesids.includes(r.id)) - .map((r) => related(r, 'passage') as string); - return supportingNotes; - }; - const sharedNotePassages = (project: ProjectD) => { - const ids = sharedNotePassageIds(project); - return ( - memory.cache.query((q) => q.findRecords('passage')) as PassageD[] - ).filter((p) => ids.includes(p.id)); - }; - const sharedNoteSections = (project: ProjectD) => { - const sectids = sharedNotePassages(project).map((p) => - related(p, 'section') - ); - return ( - memory.cache.query((q) => q.findRecords('section')) as SectionD[] - ).filter((s) => sectids.includes(s.id)); - }; - const sharedNotePlans = (project: ProjectD) => { - const planids = sharedNoteSections(project).map((p) => - related(p, 'plan') - ); - return ( - memory.cache.query((q) => q.findRecords('plan')) as PlanD[] - ).filter((s) => planids.includes(s.id)); - }; - const sharedNoteProjects = (project: ProjectD) => { - const projids = sharedNotePlans(project).map((p) => - related(p, 'project') - ); - return ( - memory.cache.query((q) => q.findRecords('project')) as ProjectD[] - ).filter((s) => projids.includes(s.id)); - }; - const AllMediafiles = (project: ProjectD) => { - const media = memory.cache.query((q) => - q.findRecords('mediafile') - ) as MediaFileD[]; - const plans = Plans(project).map((pl) => pl.id); - const planmedia = media.filter((m) => plans.includes(related(m, 'plan'))); - - //get IP media - const ip = IntellectualProperties(project, needsRemoteIds).map((i) => - related(i, 'releaseMediafile') - ); - const ipmedia = media.filter((m) => ip.includes(m.id)); - - const cats = ( - memory.cache.query((q) => - q.findRecords('artifactcategory') - ) as ArtifactCategoryD[] - ).filter( - (a) => - related(a, 'organization') === related(project, 'organization') || - related(a, 'organization') === undefined - ); - const categorymediafiles = media.filter((m) => - cats.map((c) => related(c, 'titleMediafile') as string).includes(m.id) - ); - const orgkeytermtargets = OrgKeyTermTargets(project, needsRemoteIds).map( - (i) => related(i, 'mediafile') - ); - const okttmedia = media.filter((m) => orgkeytermtargets.includes(m.id)); - - const sourcemediafiles = SourceMedia(project); - - const supportingNotePassages = sharedNotePassageIds(project); - - const sharedmedia = HighestByPassage( - media.filter((m) => - supportingNotePassages.includes(related(m, 'passage')) - ) - ); - const unique = new Set( - planmedia - .concat(ipmedia) - .concat(okttmedia) - .concat(categorymediafiles) - .concat(sourcemediafiles) - .concat(sharedmedia) - ); - return FromMedia(Array.from(unique), needsRemoteIds); - }; - const FromPassages = ( - table: string, - project: ProjectD | undefined, - remoteIds: boolean - ) => { - //passagestatechange or media - let recs = memory.cache.query((q) => q.findRecords(table)) as (BaseModel & - InitializedRecord)[]; - if (project) { - const passages = Passages(project).map((p) => p.id); - recs = recs.filter((rec) => passages.includes(related(rec, 'passage'))); - } - if (remoteIds) { - recs.forEach((r) => { - if ( - !remoteId(table, r.id, memory?.keyMap as RecordKeyMap) && - r.attributes - ) - r.attributes.offlineId = r.id; - if ( - table === 'mediafile' && - !remoteId( - 'mediafile', - related(r, 'sourceMedia'), - memory?.keyMap as RecordKeyMap - ) - ) { - (r as MediaFileD).attributes.sourceMediaOfflineId = related( - r, - 'sourceMedia' - ); - } - }); - } - return recs; - }; - const Discussions = (project: ProjectD | undefined, remoteIds: boolean) => { - let ds = memory.cache.query((q) => - q.findRecords('discussion') - ) as DiscussionD[]; - if (project) { - const mediafiles = FromPassages('mediafile', project, remoteIds).map( - (m) => m.id - ); - ds = ds.filter((rec) => mediafiles.includes(related(rec, 'mediafile'))); - } - if (remoteIds) { - ds.forEach((d) => { - if ( - !remoteId('discussion', d.id, memory?.keyMap as RecordKeyMap) && - d.attributes - ) - d.attributes.offlineId = d.id; - if ( - !remoteId( - 'mediafile', - related(d, 'mediafile'), - memory?.keyMap as RecordKeyMap - ) - ) - d.attributes.offlineMediafileId = related(d, 'mediafile'); - }); - } - return ds; - }; - const OrgKeyTerms = (project: ProjectD | undefined, remoteIds: boolean) => { - let kts = ( - memory.cache.query((q) => q.findRecords('orgkeyterm')) as OrgKeytermD[] - ).filter( - (r) => - Boolean( - remoteId( - 'organization', - related(r, 'organization'), - memory?.keyMap as RecordKeyMap - ) - ) === needsRemoteIds - ); - if (project) { - kts = kts.filter( - (rec) => - related(rec, 'organization') === related(project, 'organization') - ); - } - if (remoteIds) { - kts.forEach((kt) => { - if ( - !remoteId('orgkeyterm', kt.id, memory?.keyMap as RecordKeyMap) && - kt.attributes - ) - kt.attributes.offlineid = kt.id; - }); - } - return kts; - }; - - const OrgKeyTermTargets = ( - project: ProjectD | undefined, - remoteIds: boolean - ) => { - let ktts = ( - memory.cache.query((q) => - q.findRecords('orgkeytermtarget') - ) as OrgKeytermTargetD[] - ).filter( - (r) => - Boolean( - remoteId( - 'organization', - related(r, 'organization'), - memory?.keyMap as RecordKeyMap - ) - ) === needsRemoteIds - ); - if (project) { - ktts = ktts.filter( - (rec) => - related(rec, 'organization') === related(project, 'organization') - ); - } - if (remoteIds) { - ktts.forEach((ktt) => { - if ( - !remoteId( - 'orgkeytermtarget', - ktt.id, - memory?.keyMap as RecordKeyMap - ) && - ktt.attributes - ) - ktt.attributes.offlineId = ktt.id; - if ( - related(ktt, 'mediafile') && - !remoteId( - 'mediafile', - related(ktt, 'mediafile'), - memory?.keyMap as RecordKeyMap - ) - ) - ktt.attributes.offlineMediafileId = related(ktt, 'mediafile'); - }); - } - return ktts; - }; - const IntellectualProperties = ( - project: Project | undefined, - remoteIds: boolean - ) => { - let ips = memory.cache.query((q) => - q.findRecords('intellectualproperty') - ) as IntellectualPropertyD[]; - if (project) { - ips = ips.filter( - (rec) => - related(rec, 'organization') === related(project, 'organization') - ); - } - if (remoteIds) { - ips.forEach((ip) => { - if ( - !remoteId( - 'intellectualproperty', - ip.id, - memory?.keyMap as RecordKeyMap - ) - ) - ip.attributes.offlineId = ip.id; - if ( - !remoteId( - 'mediafile', - related(ip, 'releaseMediafile'), - memory?.keyMap as RecordKeyMap - ) - ) - ip.attributes.offlineMediafileId = related(ip, 'releaseMediafile'); - }); - } - return ips; - }; - - const FromMedia = (media: MediaFileD[], remoteIds: boolean) => { - if (remoteIds) { - media.forEach((m) => { - if ( - !remoteId('mediafile', m.id, memory?.keyMap as RecordKeyMap) && - m.attributes - ) { - m.attributes.offlineId = m.id; - } - const src = related(m, 'sourceMedia'); - if ( - src && - !remoteId('mediafile', src, memory?.keyMap as RecordKeyMap) && - m.attributes - ) { - m.attributes.sourceMediaOfflineId = src; - } - delete m.attributes.planId; - delete m.attributes.artifactTypeId; - delete m.attributes.passageId; - delete m.attributes.userId; - delete m.attributes.recordedbyUserId; - delete m.attributes.recordedByUserId; - delete m.attributes.sourceMediaId; - }); - } - return media; - }; - const Comments = (project: ProjectD | undefined, remoteIds: boolean) => { - let comments = memory.cache.query((q) => - q.findRecords('comment') - ) as BaseModelD[]; - if (project) { - const discussions = Discussions(project, remoteIds); - const discussionIds = discussions.map((d) => d.id); - comments = comments.filter((rec) => - discussionIds.includes(related(rec, 'discussion')) - ); - } - if (remoteIds) { - comments.forEach((c) => { - if ( - !remoteId('comment', c.id, memory?.keyMap as RecordKeyMap) && - c.attributes - ) { - c.attributes.offlineId = c.id; - c.attributes.offlineDiscussionId = related(c, 'discussion'); - } - if ( - !remoteId( - 'mediafile', - related(c, 'mediafile'), - memory?.keyMap as RecordKeyMap - ) && - c.attributes - ) { - c.attributes.offlineMediafileId = related(c, 'mediafile'); - } - }); - } - return comments; - }; - - const GetTableRecs = ( - info: fileInfo, - project: ProjectD | undefined, - needsRemoteIds: boolean - ): BaseModelD[] => { - const defaultQuery = (table: string) => { - return memory.cache.query((q) => q.findRecords(table)) as (BaseModel & - InitializedRecord)[]; - }; - switch (info.table) { - case 'organization': - if (project) - return [ - memory.cache.query((q) => - q.findRecord({ - type: 'organization', - id: related(project, 'organization'), - }) - ) as OrganizationD, - ]; - return defaultQuery(info.table); - - case 'project': - if (project) return [project]; - return defaultQuery(info.table); - - case 'group': - if (project) - return [ - memory.cache.query((q) => - q.findRecord({ type: 'group', id: related(project, 'group') }) - ) as GroupD, - ]; - return defaultQuery(info.table); - - case 'groupmembership': - if (project) return GroupMemberships(project); - return defaultQuery(info.table); - - case 'user': - if (project) { - const projusers = GroupMemberships(project).map((gm) => - related(gm, 'user') - ); - const users = memory.cache.query((q) => - q.findRecords(info.table) - ) as UserD[]; - - return users.filter( - (u) => projusers.find((p) => p === u.id) !== undefined - ); - } - return defaultQuery(info.table); - - case 'plan': - if (project) return Plans(project).concat(sharedNotePlans(project)); - return defaultQuery(info.table); - - case 'section': - if (project) - return Sections(project).concat(sharedNoteSections(project)); - return defaultQuery(info.table); - - case 'passage': - if (project) Passages(project).concat(sharedNotePassages(project)); - return defaultQuery(info.table); - - case 'mediafile': - if (artifactType !== undefined) { - const media = mediaArtifacts({ - memory, - projRec, - artifactType, - target, - orgWorkflowSteps, - } as IExportArtifacts); - if (media) return FromMedia(media, needsRemoteIds); - } - if (project) return AllMediafiles(project); - return FromMedia( - defaultQuery(info.table) as MediaFileD[], - needsRemoteIds - ); - - case 'passagestatechange': - return FromPassages(info.table, project, needsRemoteIds); - - case 'discussion': - return Discussions(project, needsRemoteIds); - - case 'comment': - return Comments(project, needsRemoteIds); - - case 'projectintegration': - if (project) - return memory.cache.query((q) => - q.findRecords(info.table).filter({ - relation: 'project', - record: { type: 'project', id: project.id }, - }) - ) as BaseModelD[]; - return defaultQuery(info.table); - - case 'intellectualproperty': - return IntellectualProperties(project, needsRemoteIds); - - case 'orgkeyterm': - return OrgKeyTerms(project, needsRemoteIds); + const needsRemoteIds = Boolean(projRec?.keys?.remoteId); + const { + getTableRecs: GetTableRecs, + supportingProjects, + supportingOrgs, + } = createExportCollector(memory, needsRemoteIds, { + artifactType, + target, + orgWorkflowSteps, + projRec, + organizationId: related(projRec, 'organization') || undefined, + }); - case 'orgkeytermtarget': - return OrgKeyTermTargets(project, needsRemoteIds); - default: - //activitystate,integration,plantype,projecttype,role - return defaultQuery(info.table).filter( - (r) => Boolean(r?.keys?.remoteId) === needsRemoteIds - ); - } - }; const AddChanged = async ( info: fileInfo, project: ProjectD | undefined, @@ -874,13 +321,25 @@ export async function electronExport( return 0; }; const AddSupportingProjects = async (project: ProjectD) => { - let recs = sharedNoteProjects(project); + let recs = supportingProjects(project); const ret = { Added: 0, Filtered: 0 }; - if (recs.length > 0) { + if (needsRemoteIds) { + const n = recs.length; recs = recs.filter((r) => Boolean(r.keys?.remoteId)); - ret.Added = recs?.length || 0; + ret.Filtered += n - recs.length; + } + ret.Added = recs.length; + if (recs.length > 0) await AddJsonEntry('supportingprojects', recs, 'Z'); + let orgs = supportingOrgs(project); + if (needsRemoteIds) { + const n = orgs.length; + orgs = orgs.filter((o) => Boolean(o.keys?.remoteId)); + ret.Filtered += n - orgs.length; + } + if (orgs.length > 0) { + await AddJsonEntry('supportingorgs', orgs, 'Z'); + ret.Added += orgs.length; } - AddJsonEntry('supportingprojects', recs, 'Z'); return ret; }; const AddAll = async ( @@ -900,7 +359,7 @@ export async function electronExport( ret.Filtered = len - ret.Added; } if (!scripturePackage) { - AddJsonEntry(info.table + 's', recs, info.sort); + await AddJsonEntry(info.table + 's', recs, info.sort); } switch (info.table) { case 'organization': @@ -921,11 +380,6 @@ export async function electronExport( return ret; }; - const onlyOneProject = (): boolean => { - const p = memory.cache.query((q) => q.findRecords('project')); - if (p && Array.isArray(p)) return p.length === 1; - return true; //should never get here - }; let imported: DateTime = DateTime.utc(); let op: OfflineProject | undefined; if (importedDate) { @@ -968,9 +422,7 @@ export async function electronExport( } as IBurritoMeta); await ipc?.zipAddFile(zip, 'metadata.json', burritoMetaStr, 'metadata'); } - const needsRemoteIds = Boolean(projRec?.keys?.remoteId); if (!needsRemoteIds) await AddOfflineEntry(); - const limit = onlyOneProject() ? undefined : projRec; let numRecs = 0; let numFiltered = 0; switch (expType) { @@ -979,7 +431,7 @@ export async function electronExport( case ExportType.ITFSYNC: const exported = await AddCheckEntry(); for (const info of updateableFiles) { - numRecs += await AddChanged(info, limit, needsRemoteIds); + numRecs += await AddChanged(info, projRec, needsRemoteIds); } if (expType !== ExportType.ITFBACKUP && backup) { if (!op) op = getOfflineProject(projRec.id); @@ -996,28 +448,28 @@ export async function electronExport( numRecs += ( await AddAll( { table: 'mediafile', sort: 'H' }, - limit, + projRec, needsRemoteIds, false, [ExportType.AUDIO, ExportType.ELAN].includes(expType) ) ).Added; break; - default: + default: { for (const info of updateableFiles) { - const result = await AddAll(info, limit, needsRemoteIds, true); + const result = await AddAll(info, projRec, needsRemoteIds, true); numRecs += result.Added; numFiltered += result.Filtered; } for (const info of staticFiles) { - await AddAll(info, limit, needsRemoteIds); + await AddAll(info, projRec, needsRemoteIds); } await AddFonts(); - if (limit) { - const result = await AddSupportingProjects(limit); - numRecs += result.Added; - numFiltered += result.Filtered; - } + const support = await AddSupportingProjects(projRec); + numRecs += support.Added; + numFiltered += support.Filtered; + break; + } } return { zip, numRecs, numFiltered }; }; diff --git a/src/renderer/src/store/importexport/exportTableRecs.ts b/src/renderer/src/store/importexport/exportTableRecs.ts new file mode 100644 index 000000000..4f63a1dce --- /dev/null +++ b/src/renderer/src/store/importexport/exportTableRecs.ts @@ -0,0 +1,717 @@ +import Memory from '@orbit/memory'; +import { InitializedRecord, RecordKeyMap } from '@orbit/records'; +import { + related, + remoteId, + fileInfo, + mediaArtifacts, + IExportArtifacts, +} from '../../crud'; +import { + ProjectD, + UserD, + DiscussionD, + MediaFileD, + OrganizationD, + GroupD, + GroupMembershipD, + PlanD, + SectionD, + PassageD, + ArtifactCategoryD, + SharedResourceD, + GraphicD, + OrgWorkflowStep, +} from '../../model'; +import { BaseModel, BaseModelD } from '../../model/baseModel'; + +export interface ExportCollectorOptions { + artifactType?: string | null; + target?: string; + orgWorkflowSteps?: OrgWorkflowStep[]; + projRec?: ProjectD; + organizationId?: string; +} + +export interface ExportCollector { + getTableRecs: ( + info: fileInfo, + project: ProjectD | undefined, + needsRemoteIds: boolean + ) => BaseModelD[]; + supportingProjects: (project: ProjectD) => ProjectD[]; + supportingOrgs: (project: ProjectD) => OrganizationD[]; +} + +export function createExportCollector( + memory: Memory, + needsRemoteIds: boolean, + options?: ExportCollectorOptions +): ExportCollector { + const km = memory?.keyMap as RecordKeyMap; + + const GroupMemberships = (project: ProjectD) => { + const groupid = related(project, 'group'); + return memory.cache.query((q) => + q.findRecords('groupmembership').filter({ + relation: 'group', + record: { type: 'group', id: groupid }, + }) + ) as GroupMembershipD[]; + }; + + const Plans = (project: ProjectD) => + memory.cache.query((q) => + q.findRecords('plan').filter({ + relation: 'project', + record: { type: 'project', id: project.id }, + }) + ) as PlanD[]; + + const Sections = (project: ProjectD) => { + const plans = Plans(project).map((pl) => pl.id); + const allsections = memory.cache.query((q) => + q.findRecords('section') + ) as SectionD[]; + return allsections.filter((s) => plans.includes(related(s, 'plan'))); + }; + + const Passages = (project: ProjectD) => { + const sections = Sections(project).map((s) => s.id); + const passages = memory.cache.query((q) => + q.findRecords('passage') + ) as PassageD[]; + return passages.filter((p) => sections.includes(related(p, 'section'))); + }; + + const fromIds = ( + table: string, + rel: string, + ids?: string[], + remoteIds?: boolean + ) => { + const recs = ( + memory.cache.query((q) => q.findRecords(table)) as BaseModelD[] + ).filter( + (r) => + (!ids || ids.includes(related(r, rel))) && + Boolean(r?.keys?.remoteId) === needsRemoteIds + ); + if (remoteIds) { + recs.forEach((r) => { + if (!remoteId(table, r.id, km) && r.attributes) + Object.assign(r.attributes, { offlineId: r.id }); + }); + } + return recs; + }; + + const SectionResources = (project?: ProjectD, _remoteIds?: boolean) => + fromIds( + 'sectionresource', + 'section', + project ? Sections(project).map((s) => s.id) : undefined + ); + + const SectionResourceUsers = (project?: ProjectD, remoteIds?: boolean) => + fromIds( + 'sectionresourceuser', + 'sectionresource', + SectionResources(project).map((r) => r.id), + remoteIds + ); + + const HighestByPassage = (mediafiles: MediaFileD[]) => { + const highest: MediaFileD[] = []; + let psg = ''; + mediafiles + .sort((a, b) => + related(a, 'passage') === related(b, 'passage') + ? b.attributes.versionNumber - a.attributes.versionNumber + : related(a, 'passage') > related(b, 'passage') + ? 1 + : -1 + ) + .forEach((m) => { + if (related(m, 'passage') !== psg) { + highest.push(m); + psg = related(m, 'passage'); + } + }); + return highest; + }; + + const SourceMedia = (project: ProjectD) => { + const sectionresourcemedia = SectionResources(project).map( + (r) => related(r, 'mediafile') as string + ); + const mediafiles = memory.cache.query((q) => + q.findRecords('mediafile') + ) as MediaFileD[]; + const resourcemediafiles = mediafiles.filter((m) => + sectionresourcemedia.includes(m.id) + ); + const sourcemediafiles = mediafiles.filter( + (m) => + m.attributes?.readyToShare && + resourcemediafiles + .map((r) => related(r, 'resourcePassage')) + .includes(related(m, 'passage')) + ); + return HighestByPassage(sourcemediafiles); + }; + + const sharedNotePassageIds = (project: ProjectD) => { + const psgs = Passages(project).filter( + (p) => related(p, 'sharedResource') !== null + ); + const sharednotesids = psgs.map( + (p) => related(p, 'sharedResource') as string + ); + return ( + memory.cache.query((q) => + q.findRecords('sharedresource') + ) as SharedResourceD[] + ) + .filter((r) => sharednotesids.includes(r.id)) + .map((r) => related(r, 'passage') as string); + }; + + const sharedNotePassages = (project: ProjectD) => { + const ids = sharedNotePassageIds(project); + return ( + memory.cache.query((q) => q.findRecords('passage')) as PassageD[] + ).filter((p) => ids.includes(p.id)); + }; + + const SharedResources = (project?: ProjectD, _remoteIds?: boolean) => + fromIds( + 'sharedresource', + 'passage', + project + ? Passages(project) + .concat(sharedNotePassages(project)) + .map((p) => p.id) + : undefined + ); + + const SharedResourceReferences = (project?: ProjectD, _remoteIds?: boolean) => + fromIds( + 'sharedresourcereference', + 'sharedResource', + SharedResources(project).map((r) => r.id) + ); + + const sharedNoteSections = (project: ProjectD) => { + const sectids = sharedNotePassages(project).map((p) => + related(p, 'section') + ); + return ( + memory.cache.query((q) => q.findRecords('section')) as SectionD[] + ).filter((s) => sectids.includes(s.id)); + }; + + const sharedNotePlans = (project: ProjectD) => { + const planids = sharedNoteSections(project).map((p) => related(p, 'plan')); + return (memory.cache.query((q) => q.findRecords('plan')) as PlanD[]).filter( + (s) => planids.includes(s.id) + ); + }; + + const supportingProjects = (project: ProjectD) => { + const projids = sharedNotePlans(project).map((p) => related(p, 'project')); + return ( + memory.cache.query((q) => q.findRecords('project')) as ProjectD[] + ).filter((s) => projids.includes(s.id)); + }; + + const sharedNoteArtifactCategories = ( + project: ProjectD, + _remoteIds: boolean + ) => { + const projOrgId = related(project, 'organization'); + const sharedResources = SharedResources(project) as SharedResourceD[]; + const foreignCatIds = [ + ...new Set( + sharedResources + .map((r) => related(r, 'artifactCategory') as string) + .filter(Boolean) + ), + ]; + if (foreignCatIds.length === 0) return [] as ArtifactCategoryD[]; + return ( + memory.cache.query((q) => + q.findRecords('artifactcategory') + ) as ArtifactCategoryD[] + ).filter((a) => { + const org = related(a, 'organization'); + return ( + foreignCatIds.includes(a.id) && + Boolean(org) && + org !== projOrgId && + Boolean(remoteId('organization', org, km)) === needsRemoteIds + ); + }); + }; + + const supportingOrgs = (project: ProjectD) => { + const projOrgId = related(project, 'organization'); + const cats = sharedNoteArtifactCategories(project, needsRemoteIds); + const orgIds = [ + ...new Set( + cats + .map((c) => related(c, 'organization')) + .concat( + supportingProjects(project).map((p) => related(p, 'organization')) + ) + .filter((id) => id && id !== projOrgId) + ), + ]; + return ( + memory.cache.query((q) => + q.findRecords('organization') + ) as OrganizationD[] + ).filter( + (o) => + orgIds.includes(o.id) && + Boolean(remoteId('organization', o.id, km)) === needsRemoteIds + ); + }; + + const orgTable = ( + table: string, + project: ProjectD | undefined, + remoteIds: boolean, + { + rel = 'organization', + mediafile, + }: { rel?: string; mediafile?: string } = {} + ) => { + const scopeId = + rel === 'project' + ? project?.id + : (options?.organizationId ?? + (project ? related(project, 'organization') : undefined)); + let recs = ( + memory.cache.query((q) => q.findRecords(table)) as BaseModelD[] + ).filter((r) => { + const id = related(r, rel); + return Boolean(id) && Boolean(remoteId(rel, id, km)) === needsRemoteIds; + }); + if (scopeId) recs = recs.filter((rec) => related(rec, rel) === scopeId); + if (remoteIds) { + recs.forEach((r) => { + if (!remoteId(table, r.id, km) && r.attributes) + Object.assign(r.attributes, { offlineId: r.id }); + const mid = mediafile ? related(r, mediafile) : ''; + if (mid && !remoteId('mediafile', mid, km) && r.attributes) + Object.assign(r.attributes, { offlineMediafileId: mid }); + }); + } + return recs; + }; + + const ArtifactCategories = ( + project: ProjectD | undefined, + remoteIds: boolean + ) => { + const cats = orgTable('artifactcategory', project, remoteIds); + if (!project) return cats; + const globals = ( + memory.cache.query((q) => + q.findRecords('artifactcategory') + ) as ArtifactCategoryD[] + ).filter((a) => { + const org = related(a, 'organization'); + return ( + (org === null || org === undefined) && + Boolean(a?.keys?.remoteId) === needsRemoteIds + ); + }); + if (remoteIds) { + globals.forEach((r) => { + if (!remoteId('artifactcategory', r.id, km) && r.attributes) + Object.assign(r.attributes, { offlineId: r.id }); + }); + } + const seen = new Set(cats.map((c) => c.id)); + return cats.concat( + globals + .concat(sharedNoteArtifactCategories(project, remoteIds)) + .filter((c) => { + if (seen.has(c.id)) return false; + seen.add(c.id); + return true; + }) + ); + }; + + const Graphics = (project: ProjectD | undefined, remoteIds: boolean) => { + const recs = orgTable('graphic', project, remoteIds) as GraphicD[]; + const sections = project + ? Sections(project).concat(sharedNoteSections(project)) + : (memory.cache.query((q) => q.findRecords('section')) as SectionD[]); + const passages = project + ? Passages(project).concat(sharedNotePassages(project)) + : (memory.cache.query((q) => q.findRecords('passage')) as PassageD[]); + const remoteNums = (kind: string, rows: { id: string }[]) => + new Set( + rows + .map((r) => parseInt(remoteId(kind, r.id, km) ?? '', 10)) + .filter((n) => !Number.isNaN(n)) + ); + const sectionIds = remoteNums('section', sections); + const passageIds = remoteNums('passage', passages); + return recs.filter((g) => { + const t = g.attributes?.resourceType; + const id = g.attributes?.resourceId; + return ( + t === 'category' || + (t === 'section' && sectionIds.has(id)) || + (t === 'passage' && passageIds.has(id)) + ); + }); + }; + + const OrganizationSchemeSteps = ( + project: ProjectD | undefined, + remoteIds: boolean + ) => + fromIds( + 'organizationschemestep', + 'organizationscheme', + orgTable('organizationscheme', project, remoteIds).map((s) => s.id) + ); + + const FromMedia = (mediafiles: MediaFileD[], remoteIds: boolean) => { + if (remoteIds) { + mediafiles.forEach((m) => { + if (!remoteId('mediafile', m.id, km) && m.attributes) { + m.attributes.offlineId = m.id; + } + const src = related(m, 'sourceMedia'); + if (src && !remoteId('mediafile', src, km) && m.attributes) { + m.attributes.sourceMediaOfflineId = src; + } + delete m.attributes.planId; + delete m.attributes.artifactTypeId; + delete m.attributes.passageId; + delete m.attributes.userId; + delete m.attributes.recordedbyUserId; + delete m.attributes.recordedByUserId; + delete m.attributes.sourceMediaId; + }); + } + return mediafiles; + }; + + const AllMediafiles = (project: ProjectD) => { + const mediafiles = memory.cache.query((q) => + q.findRecords('mediafile') + ) as MediaFileD[]; + const plans = Plans(project).map((pl) => pl.id); + const planmedia = mediafiles.filter((m) => + plans.includes(related(m, 'plan')) + ); + const ip = orgTable('intellectualproperty', project, needsRemoteIds, { + mediafile: 'releaseMediafile', + }).map((i) => related(i, 'releaseMediafile')); + const ipmedia = mediafiles.filter((m) => ip.includes(m.id)); + const catTitleIds = ArtifactCategories(project, needsRemoteIds).map( + (c) => related(c, 'titleMediafile') as string + ); + const categorymediafiles = mediafiles.filter((m) => + catTitleIds.includes(m.id) + ); + const orgkeytermtargets = orgTable( + 'orgkeytermtarget', + project, + needsRemoteIds, + { mediafile: 'mediafile' } + ).map((i) => related(i, 'mediafile')); + const okttmedia = mediafiles.filter((m) => + orgkeytermtargets.includes(m.id) + ); + const graphicmedia = Graphics(project, needsRemoteIds).map((g) => + related(g, 'mediafile') + ); + const grmedia = mediafiles.filter((m) => graphicmedia.includes(m.id)); + const sourcemediafiles = SourceMedia(project); + const supportingNotePassages = sharedNotePassageIds(project); + const sharedmedia = HighestByPassage( + mediafiles.filter((m) => + supportingNotePassages.includes(related(m, 'passage')) + ) + ); + const unique = new Set( + planmedia + .concat(ipmedia) + .concat(okttmedia) + .concat(categorymediafiles) + .concat(sourcemediafiles) + .concat(sharedmedia) + .concat(grmedia) + ); + return FromMedia(Array.from(unique), needsRemoteIds); + }; + + const FromPassages = ( + table: string, + project: ProjectD | undefined, + remoteIds: boolean + ) => { + let recs = memory.cache.query((q) => q.findRecords(table)) as (BaseModel & + InitializedRecord)[]; + if (project) { + const passages = Passages(project).map((p) => p.id); + recs = recs.filter((rec) => passages.includes(related(rec, 'passage'))); + } + if (remoteIds) { + recs.forEach((r) => { + if (!remoteId(table, r.id, km) && r.attributes) + r.attributes.offlineId = r.id; + if ( + table === 'mediafile' && + !remoteId('mediafile', related(r, 'sourceMedia'), km) + ) { + (r as MediaFileD).attributes.sourceMediaOfflineId = related( + r, + 'sourceMedia' + ); + } + }); + } + return recs; + }; + + const Discussions = (project: ProjectD | undefined, remoteIds: boolean) => { + let ds = memory.cache.query((q) => + q.findRecords('discussion') + ) as DiscussionD[]; + if (project) { + const mediafiles = FromPassages('mediafile', project, remoteIds).map( + (m) => m.id + ); + ds = ds.filter((rec) => mediafiles.includes(related(rec, 'mediafile'))); + } + if (remoteIds) { + ds.forEach((d) => { + if (!remoteId('discussion', d.id, km) && d.attributes) + d.attributes.offlineId = d.id; + if (!remoteId('mediafile', related(d, 'mediafile'), km)) + d.attributes.offlineMediafileId = related(d, 'mediafile'); + }); + } + return ds; + }; + + const Comments = (project: ProjectD | undefined, remoteIds: boolean) => { + let comments = memory.cache.query((q) => + q.findRecords('comment') + ) as BaseModelD[]; + if (project) { + const discussions = Discussions(project, remoteIds); + const discussionIds = discussions.map((d) => d.id); + comments = comments.filter((rec) => + discussionIds.includes(related(rec, 'discussion')) + ); + } + if (remoteIds) { + comments.forEach((c) => { + if (!remoteId('comment', c.id, km) && c.attributes) { + c.attributes.offlineId = c.id; + c.attributes.offlineDiscussionId = related(c, 'discussion'); + } + if (!remoteId('mediafile', related(c, 'mediafile'), km) && c.attributes) + c.attributes.offlineMediafileId = related(c, 'mediafile'); + }); + } + return comments; + }; + + const defaultQuery = (table: string) => + memory.cache.query((q) => q.findRecords(table)) as (BaseModel & + InitializedRecord)[]; + + const getTableRecs = ( + info: fileInfo, + project: ProjectD | undefined, + remoteIds: boolean + ): BaseModelD[] => { + switch (info.table) { + case 'organization': + if (project) + return [ + memory.cache.query((q) => + q.findRecord({ + type: 'organization', + id: related(project, 'organization'), + }) + ) as OrganizationD, + ]; + return defaultQuery(info.table); + + case 'project': + if (project) return [project]; + return defaultQuery(info.table); + + case 'group': + if (project) + return [ + memory.cache.query((q) => + q.findRecord({ type: 'group', id: related(project, 'group') }) + ) as GroupD, + ]; + return defaultQuery(info.table); + + case 'groupmembership': + if (project) return GroupMemberships(project); + return defaultQuery(info.table); + + case 'user': + if (project) { + const projusers = GroupMemberships(project).map((gm) => + related(gm, 'user') + ); + const users = memory.cache.query((q) => + q.findRecords(info.table) + ) as UserD[]; + return users.filter( + (u) => projusers.find((p) => p === u.id) !== undefined + ); + } + return defaultQuery(info.table); + + case 'plan': + if (project) return Plans(project).concat(sharedNotePlans(project)); + return defaultQuery(info.table); + + case 'section': + if (project) + return Sections(project).concat(sharedNoteSections(project)); + return defaultQuery(info.table); + + case 'passage': + if (project) + return Passages(project).concat(sharedNotePassages(project)); + return defaultQuery(info.table); + + case 'mediafile': + if (options?.artifactType !== undefined && options.projRec) { + const mediafiles = mediaArtifacts({ + memory, + projRec: options.projRec, + artifactType: options.artifactType, + target: options.target, + orgWorkflowSteps: options.orgWorkflowSteps, + } as IExportArtifacts); + if (mediafiles) return FromMedia(mediafiles, remoteIds); + } + if (project) return AllMediafiles(project); + if (options?.organizationId) { + const ipIds = orgTable('intellectualproperty', undefined, remoteIds, { + mediafile: 'releaseMediafile', + }).map((i) => related(i, 'releaseMediafile')); + const mediafiles = (defaultQuery('mediafile') as MediaFileD[]).filter( + (m) => ipIds.includes(m.id) + ); + return FromMedia(mediafiles, remoteIds); + } + return FromMedia(defaultQuery(info.table) as MediaFileD[], remoteIds); + + case 'passagestatechange': + return FromPassages(info.table, project, remoteIds); + + case 'discussion': + return Discussions(project, remoteIds); + + case 'comment': + return Comments(project, remoteIds); + + case 'projectintegration': + if (project) + return memory.cache.query((q) => + q.findRecords(info.table).filter({ + relation: 'project', + record: { type: 'project', id: project.id }, + }) + ) as BaseModelD[]; + return defaultQuery(info.table); + + case 'intellectualproperty': + return orgTable('intellectualproperty', project, remoteIds, { + mediafile: 'releaseMediafile', + }); + + case 'orgkeyterm': + return orgTable('orgkeyterm', project, remoteIds); + + case 'orgkeytermtarget': + return orgTable('orgkeytermtarget', project, remoteIds, { + mediafile: 'mediafile', + }); + case 'orgkeytermreference': + return orgTable('orgkeytermreference', project, remoteIds, { + rel: 'project', + }); + case 'sectionresourceuser': + return SectionResourceUsers(project, remoteIds); + case 'organizationscheme': + return orgTable('organizationscheme', project, remoteIds); + case 'organizationschemestep': + return OrganizationSchemeSteps(project, remoteIds); + case 'graphic': + return Graphics(project, remoteIds); + case 'organizationmembership': + return orgTable('organizationmembership', project, remoteIds); + case 'orgworkflowstep': + return orgTable('orgworkflowstep', project, remoteIds); + case 'sectionresource': + return SectionResources(project, remoteIds); + case 'sharedresource': + return SharedResources(project, remoteIds); + case 'sharedresourcereference': + return SharedResourceReferences(project, remoteIds); + case 'artifactcategory': + return ArtifactCategories(project, remoteIds); + + case 'organizationbible': + return orgTable('organizationbible', project, remoteIds); + + case 'bible': + if (project) { + const orgBibleIds = orgTable( + 'organizationbible', + project, + remoteIds + ).map((ob) => related(ob, 'bible')); + return defaultQuery('bible').filter( + (b) => + orgBibleIds.includes(b.id) && + Boolean(b?.keys?.remoteId) === needsRemoteIds + ); + } + return defaultQuery(info.table).filter( + (r) => Boolean(r?.keys?.remoteId) === needsRemoteIds + ); + + case 'activitystate': + case 'artifacttype': + case 'integration': + case 'passagetype': + case 'plantype': + case 'projecttype': + case 'role': + case 'workflowstep': + return defaultQuery(info.table).filter( + (r) => Boolean(r?.keys?.remoteId) === needsRemoteIds + ); + case 'invitation': + return []; + default: + throw new Error(`GetTableRecs: unhandled table '${info.table}'`); + } + }; + + return { getTableRecs, supportingProjects, supportingOrgs }; +} diff --git a/src/renderer/src/store/importexport/projectDataExport.test.ts b/src/renderer/src/store/importexport/projectDataExport.test.ts new file mode 100644 index 000000000..dc34aade9 --- /dev/null +++ b/src/renderer/src/store/importexport/projectDataExport.test.ts @@ -0,0 +1,589 @@ +import type Memory from '@orbit/memory'; +import type { ProjectD } from '../../model'; +import { related } from '../../crud/related'; + +jest.mock('../../serializers/getSerializer', () => ({ + getSerializer: () => ({ + serialize: (r: { type: string; id: string; attributes?: unknown }) => ({ + type: r.type, + id: r.id, + attributes: r.attributes, + }), + }), +})); + +jest.mock('../../crud', () => ({ + related: jest.requireActual('../../crud/related').related, + remoteId: jest.requireActual('../../crud/remoteId').remoteId, + updateableFiles: jest.requireActual('../../crud/fileOrder').updateableFiles, + staticFiles: jest.requireActual('../../crud/fileOrder').staticFiles, + mediaArtifacts: jest.fn(), +})); + +import { getProjectDataFiles } from './projectDataExport'; +import { createExportCollector } from './exportTableRecs'; + +const rel = (type: string, id: string) => ({ data: { type, id } }); + +const orgRec = (id: string, name: string, keyed = true) => ({ + type: 'organization', + id, + ...(keyed ? { keys: { remoteId: id } } : {}), + attributes: { + name, + dateCreated: '2020-01-01T00:00:00.000Z', + dateUpdated: '2020-01-01T00:00:00.000Z', + }, + relationships: {}, +}); + +const orgScoped = (type: string, id: string, orgId: string, keyed = true) => ({ + type, + id, + ...(keyed ? { keys: { remoteId: id } } : {}), + attributes: { + dateCreated: '2020-01-01T00:00:00.000Z', + dateUpdated: '2020-01-01T00:00:00.000Z', + }, + relationships: { organization: rel('organization', orgId) }, +}); + +function memoryStub(store: Record): Memory { + return { + schema: {}, + keyMap: { + idToKey: (table: string, _key: string, localId: string) => { + if (localId == null) + throw new Error(`idToKey called without localId (${table})`); + const rec = ( + (store[table] ?? []) as { id: string; keys?: { remoteId?: string } }[] + ).find((r) => r.id === localId); + return rec?.keys?.remoteId; + }, + }, + cache: { + query: (cb: (q: unknown) => unknown) => { + const q = { + findRecords: (type: string) => { + const recs = [...((store[type] ?? []) as { id: string }[])]; + const origFilter = recs.filter.bind(recs); + ( + recs as unknown as { + filter: (f: unknown) => unknown; + } + ).filter = (f: unknown) => { + if (typeof f === 'function') + return origFilter(f as (r: { id: string }) => boolean); + const spec = f as { relation: string; record: { id: string } }; + return origFilter( + (r) => related(r, spec.relation) === spec.record.id + ); + }; + return recs; + }, + findRecord: (ident: { type: string; id: string }) => ident, + }; + const spec = cb(q) as { type?: string; id?: string } | unknown[]; + if (Array.isArray(spec)) return spec; + if (spec?.type && spec?.id) + return ((store[spec.type] ?? []) as { id: string }[]).find( + (r) => r.id === spec.id + ); + return spec; + }, + }, + } as unknown as Memory; +} + +const idsIn = (files: Record, filename: string) => { + const json = files[filename]; + if (!json) return [] as string[]; + return (JSON.parse(json).data as { id: string }[]).map((r) => r.id); +}; + +describe('getProjectDataFiles project scoping', () => { + const project = { + type: 'project', + id: 'proj-mine', + keys: { remoteId: 'proj-mine' }, + attributes: { + name: 'Mine', + dateCreated: '2020-01-01T00:00:00.000Z', + dateUpdated: '2020-01-01T00:00:00.000Z', + }, + relationships: { + organization: rel('organization', 'org-mine'), + group: rel('group', 'group-mine'), + }, + } as unknown as ProjectD; + + const offlineProject = { + type: 'project', + id: 'proj-local', + attributes: { + name: 'Local', + dateCreated: '2020-01-01T00:00:00.000Z', + dateUpdated: '2020-01-01T00:00:00.000Z', + }, + relationships: { + organization: rel('organization', 'org-local'), + group: rel('group', 'group-mine'), + }, + } as unknown as ProjectD; + + const store: Record = { + organization: [ + orgRec('org-mine', 'Mine Org'), + orgRec('org-other', 'Other Org'), + orgRec('org-local', 'Local Org', false), + ], + group: [ + { + type: 'group', + id: 'group-mine', + keys: { remoteId: 'group-mine' }, + attributes: { + name: 'G', + dateCreated: '2020-01-01T00:00:00.000Z', + dateUpdated: '2020-01-01T00:00:00.000Z', + }, + relationships: {}, + }, + ], + project: [project, offlineProject], + groupmembership: [], + user: [], + intellectualproperty: [ + orgScoped('intellectualproperty', 'ip-mine', 'org-mine'), + orgScoped('intellectualproperty', 'ip-mine-local', 'org-mine', false), + orgScoped('intellectualproperty', 'ip-other', 'org-other'), + orgScoped('intellectualproperty', 'ip-local', 'org-local', false), + ], + orgworkflowstep: [ + orgScoped('orgworkflowstep', 'ows-mine', 'org-mine'), + orgScoped('orgworkflowstep', 'ows-other', 'org-other'), + orgScoped('orgworkflowstep', 'ows-local', 'org-local', false), + ], + organizationbible: [ + orgScoped('organizationbible', 'ob-mine', 'org-mine'), + orgScoped('organizationbible', 'ob-other', 'org-other'), + orgScoped('organizationbible', 'ob-local', 'org-local', false), + ], + orgkeyterm: [ + orgScoped('orgkeyterm', 'okt-mine', 'org-mine'), + orgScoped('orgkeyterm', 'okt-mine-local', 'org-mine', false), + orgScoped('orgkeyterm', 'okt-other', 'org-other'), + orgScoped('orgkeyterm', 'okt-local', 'org-local', false), + ], + artifactcategory: [ + { + type: 'artifactcategory', + id: 'cat-global', + keys: { remoteId: 'cat-global' }, + attributes: { + dateCreated: '2020-01-01T00:00:00.000Z', + dateUpdated: '2020-01-01T00:00:00.000Z', + }, + relationships: {}, + }, + { + type: 'artifactcategory', + id: 'cat-global-local', + attributes: { + dateCreated: '2020-01-01T00:00:00.000Z', + dateUpdated: '2020-01-01T00:00:00.000Z', + }, + relationships: { organization: { data: null } }, + }, + ], + }; + + it('omits other-org and unkeyed rows from an online project export', async () => { + const memory = memoryStub(store); + const files = await getProjectDataFiles(memory, project); + + expect(idsIn(files, 'data/B_organizations.json')).toEqual(['org-mine']); + expect(idsIn(files, 'data/I_intellectualpropertys.json')).toEqual([ + 'ip-mine', + ]); + expect(idsIn(files, 'data/C_orgworkflowsteps.json')).toEqual(['ows-mine']); + expect(idsIn(files, 'data/J_organizationbibles.json')).toEqual(['ob-mine']); + expect(idsIn(files, 'data/C_orgkeyterms.json')).toEqual(['okt-mine']); + expect(idsIn(files, 'data/C_artifactcategorys.json')).toEqual([ + 'cat-global', + ]); + expect(idsIn(files, 'data/D_projects.json')).toEqual(['proj-mine']); + }); + + it('keeps unkeyed org-scoped rows in an offline project export', async () => { + const files = await getProjectDataFiles(memoryStub(store), offlineProject); + + expect(idsIn(files, 'data/B_organizations.json')).toEqual(['org-local']); + expect(idsIn(files, 'data/I_intellectualpropertys.json')).toEqual([ + 'ip-local', + ]); + expect(idsIn(files, 'data/C_orgworkflowsteps.json')).toEqual(['ows-local']); + expect(idsIn(files, 'data/J_organizationbibles.json')).toEqual([ + 'ob-local', + ]); + expect(idsIn(files, 'data/C_orgkeyterms.json')).toEqual(['okt-local']); + expect(idsIn(files, 'data/C_artifactcategorys.json')).toEqual([ + 'cat-global-local', + ]); + expect(idsIn(files, 'data/D_projects.json')).toEqual(['proj-local']); + }); + + it('unscoped getTableRecs follows the related-org remote cohort', () => { + const memory = memoryStub(store); + const online = createExportCollector(memory, true); + expect( + online + .getTableRecs( + { table: 'intellectualproperty', sort: 'I' }, + undefined, + true + ) + .map((r) => r.id) + .sort() + ).toEqual(['ip-mine', 'ip-mine-local', 'ip-other']); + + const offline = createExportCollector(memory, false); + expect( + offline + .getTableRecs( + { table: 'intellectualproperty', sort: 'I' }, + undefined, + false + ) + .map((r) => r.id) + ).toEqual(['ip-local']); + }); +}); + +describe('supportingOrgs from supporting projects', () => { + const dates = { + dateCreated: '2020-01-01T00:00:00.000Z', + dateUpdated: '2020-01-01T00:00:00.000Z', + }; + + const project = { + type: 'project', + id: 'proj-mine', + keys: { remoteId: 'proj-mine' }, + attributes: { name: 'Mine', ...dates }, + relationships: { + organization: rel('organization', 'org-mine'), + group: rel('group', 'group-mine'), + }, + } as unknown as ProjectD; + + const projSrc = { + type: 'project', + id: 'proj-src', + keys: { remoteId: 'proj-src' }, + attributes: { name: 'Source', ...dates }, + relationships: { + organization: rel('organization', 'org-src'), + group: rel('group', 'group-src'), + }, + }; + + const store: Record = { + organization: [ + orgRec('org-mine', 'Mine Org'), + orgRec('org-src', 'Source Org'), + ], + group: [ + { + type: 'group', + id: 'group-mine', + keys: { remoteId: 'group-mine' }, + attributes: { name: 'G', ...dates }, + relationships: {}, + }, + ], + project: [project, projSrc], + plan: [ + { + type: 'plan', + id: 'plan-mine', + keys: { remoteId: 'plan-mine' }, + attributes: dates, + relationships: { project: rel('project', 'proj-mine') }, + }, + { + type: 'plan', + id: 'plan-src', + keys: { remoteId: 'plan-src' }, + attributes: dates, + relationships: { project: rel('project', 'proj-src') }, + }, + ], + section: [ + { + type: 'section', + id: 'sec-mine', + keys: { remoteId: 'sec-mine' }, + attributes: dates, + relationships: { plan: rel('plan', 'plan-mine') }, + }, + { + type: 'section', + id: 'sec-src', + keys: { remoteId: 'sec-src' }, + attributes: dates, + relationships: { plan: rel('plan', 'plan-src') }, + }, + ], + passage: [ + { + type: 'passage', + id: 'pas-mine', + keys: { remoteId: 'pas-mine' }, + attributes: dates, + relationships: { + section: rel('section', 'sec-mine'), + sharedResource: rel('sharedresource', 'sr-1'), + }, + }, + { + type: 'passage', + id: 'pas-src', + keys: { remoteId: 'pas-src' }, + attributes: dates, + relationships: { section: rel('section', 'sec-src') }, + }, + ], + sharedresource: [ + { + type: 'sharedresource', + id: 'sr-1', + keys: { remoteId: 'sr-1' }, + attributes: dates, + relationships: { passage: rel('passage', 'pas-src') }, + }, + ], + }; + + it('includes the supporting project org when the shared resource has no artifact category', async () => { + const memory = memoryStub(store); + const { supportingOrgs, supportingProjects } = createExportCollector( + memory, + true, + { projRec: project, organizationId: 'org-mine' } + ); + + expect(supportingProjects(project).map((p) => p.id)).toEqual(['proj-src']); + expect(supportingOrgs(project).map((o) => o.id)).toEqual(['org-src']); + + const files = await getProjectDataFiles(memory, project); + expect(idsIn(files, 'data/Z_supportingorgs.json')).toEqual(['org-src']); + expect(idsIn(files, 'data/Z_supportingprojects.json')).toEqual([ + 'proj-src', + ]); + expect(idsIn(files, 'data/B_organizations.json')).toEqual(['org-mine']); + }); + + it('includes title media for a shared-note category owned by the supporting org', async () => { + const withTitle = { + ...store, + artifactcategory: [ + { + type: 'artifactcategory', + id: 'cat-src', + keys: { remoteId: 'cat-src' }, + attributes: dates, + relationships: { + organization: rel('organization', 'org-src'), + titleMediafile: rel('mediafile', 'media-title'), + }, + }, + ], + sharedresource: [ + { + type: 'sharedresource', + id: 'sr-1', + keys: { remoteId: 'sr-1' }, + attributes: dates, + relationships: { + passage: rel('passage', 'pas-src'), + artifactCategory: rel('artifactcategory', 'cat-src'), + }, + }, + ], + mediafile: [ + { + type: 'mediafile', + id: 'media-title', + keys: { remoteId: 'media-title' }, + attributes: { ...dates, audioUrl: 'title.mp3', versionNumber: 1 }, + relationships: {}, + }, + ], + }; + const memory = memoryStub(withTitle); + const { getTableRecs } = createExportCollector(memory, true, { + projRec: project, + organizationId: 'org-mine', + }); + const media = getTableRecs( + { table: 'mediafile', sort: 'H' }, + project, + true + ); + expect(media.map((m) => m.id)).toContain('media-title'); + + const files = await getProjectDataFiles(memory, project); + expect(idsIn(files, 'data/C_artifactcategorys.json')).toContain('cat-src'); + expect(idsIn(files, 'data/H_mediafiles.json')).toContain('media-title'); + }); + + it('exports only the highest version of shared-note source media', () => { + const withVersions = { + ...store, + mediafile: [ + { + type: 'mediafile', + id: 'media-v1', + keys: { remoteId: 'media-v1' }, + attributes: { ...dates, versionNumber: 1 }, + relationships: { passage: rel('passage', 'pas-src') }, + }, + { + type: 'mediafile', + id: 'media-v3', + keys: { remoteId: 'media-v3' }, + attributes: { ...dates, versionNumber: 3 }, + relationships: { passage: rel('passage', 'pas-src') }, + }, + { + type: 'mediafile', + id: 'media-v2', + keys: { remoteId: 'media-v2' }, + attributes: { ...dates, versionNumber: 2 }, + relationships: { passage: rel('passage', 'pas-src') }, + }, + ], + }; + const { getTableRecs } = createExportCollector( + memoryStub(withVersions), + true, + { projRec: project, organizationId: 'org-mine' } + ); + const media = getTableRecs( + { table: 'mediafile', sort: 'H' }, + project, + true + ); + expect(media.map((m) => m.id)).toEqual(['media-v3']); + }); +}); + +describe('fromIds remote identity cohort', () => { + const dates = { + dateCreated: '2020-01-01T00:00:00.000Z', + dateUpdated: '2020-01-01T00:00:00.000Z', + }; + + const onlineProject = { + type: 'project', + id: 'proj-mine', + keys: { remoteId: 'proj-mine' }, + attributes: { name: 'Mine', ...dates }, + relationships: { + organization: rel('organization', 'org-mine'), + group: rel('group', 'group-mine'), + }, + } as unknown as ProjectD; + + const offlineProject = { + ...onlineProject, + keys: undefined, + } as unknown as ProjectD; + + const store: Record = { + organization: [orgRec('org-mine', 'Mine Org')], + group: [ + { + type: 'group', + id: 'group-mine', + keys: { remoteId: 'group-mine' }, + attributes: { name: 'G', ...dates }, + relationships: {}, + }, + ], + project: [onlineProject], + plan: [ + { + type: 'plan', + id: 'plan-mine', + keys: { remoteId: 'plan-mine' }, + attributes: dates, + relationships: { project: rel('project', 'proj-mine') }, + }, + ], + section: [ + { + type: 'section', + id: 'sec-mine', + keys: { remoteId: 'sec-mine' }, + attributes: dates, + relationships: { plan: rel('plan', 'plan-mine') }, + }, + ], + passage: [ + { + type: 'passage', + id: 'pas-mine', + keys: { remoteId: 'pas-mine' }, + attributes: dates, + relationships: { section: rel('section', 'sec-mine') }, + }, + ], + sectionresource: [ + { + type: 'sectionresource', + id: 'sr-remote', + keys: { remoteId: 'sr-remote' }, + attributes: dates, + relationships: { section: rel('section', 'sec-mine') }, + }, + { + type: 'sectionresource', + id: 'sr-local', + attributes: dates, + relationships: { section: rel('section', 'sec-mine') }, + }, + ], + sharedresource: [ + { + type: 'sharedresource', + id: 'shr-remote', + keys: { remoteId: 'shr-remote' }, + attributes: dates, + relationships: { passage: rel('passage', 'pas-mine') }, + }, + { + type: 'sharedresource', + id: 'shr-local', + attributes: dates, + relationships: { passage: rel('passage', 'pas-mine') }, + }, + ], + }; + + it('omits local-only static rows from an online project export', async () => { + const files = await getProjectDataFiles(memoryStub(store), onlineProject); + expect(idsIn(files, 'data/I_sectionresources.json')).toEqual(['sr-remote']); + expect(idsIn(files, 'data/I_sharedresources.json')).toEqual(['shr-remote']); + }); + + it('keeps local-only static rows in an offline project export', async () => { + const files = await getProjectDataFiles( + memoryStub({ ...store, project: [offlineProject] }), + offlineProject + ); + expect(idsIn(files, 'data/I_sectionresources.json')).toEqual(['sr-local']); + expect(idsIn(files, 'data/I_sharedresources.json')).toEqual(['shr-local']); + }); +}); diff --git a/src/renderer/src/store/importexport/projectDataExport.ts b/src/renderer/src/store/importexport/projectDataExport.ts index 0ac76c17c..075c334a4 100644 --- a/src/renderer/src/store/importexport/projectDataExport.ts +++ b/src/renderer/src/store/importexport/projectDataExport.ts @@ -1,36 +1,13 @@ /** - * Exports project data in PTF format (data/*.json) for use in burrito creation - * or other contexts. Mirrors the data folder structure from electronExport. + * Exports project data in PTF format (data/*.json) for burrito creation + * or other contexts. Uses the same table selection as electronExport. */ import Memory from '@orbit/memory'; import { getSerializer } from '../../serializers/getSerializer'; -import { InitializedRecord, RecordKeyMap } from '@orbit/records'; -import { - related, - remoteId, - fileInfo, - updateableFiles, - staticFiles, -} from '../../crud'; -import { - ProjectD, - UserD, - DiscussionD, - OrgKeytermD, - OrgKeytermTargetD, - IntellectualPropertyD, - MediaFileD, - OrganizationD, - GroupD, - GroupMembershipD, - PlanD, - SectionD, - PassageD, - SectionResourceD, - ArtifactCategoryD, - SharedResourceD, -} from '../../model'; -import { BaseModel, BaseModelD } from '../../model/baseModel'; +import { InitializedRecord } from '@orbit/records'; +import { related, updateableFiles, staticFiles } from '../../crud'; +import { ProjectD, MediaFileD, OrganizationD } from '../../model'; +import { createExportCollector } from './exportTableRecs'; export interface ProjectDataFiles { [filename: string]: string; @@ -38,10 +15,10 @@ export interface ProjectDataFiles { function serializeRecords( recs: InitializedRecord[], - projRec: ProjectD, + needsRemoteIds: boolean, ser: ReturnType ): Record[] { - return projRec?.keys?.remoteId + return needsRemoteIds ? (recs.map((r) => ser.serialize(r)) as unknown as Record< string, unknown @@ -59,512 +36,44 @@ export async function getProjectDataFiles( project: ProjectD ): Promise { const ser = getSerializer(memory); - const keyMap = memory?.keyMap as RecordKeyMap; const needsRemoteIds = Boolean(project?.keys?.remoteId); + const { getTableRecs, supportingProjects, supportingOrgs } = + createExportCollector(memory, needsRemoteIds, { + projRec: project, + organizationId: related(project, 'organization') || undefined, + }); const files: ProjectDataFiles = {}; - const GroupMemberships = (proj: ProjectD) => { - const groupid = related(proj, 'group'); - return memory.cache.query((q) => - q.findRecords('groupmembership').filter({ - relation: 'group', - record: { type: 'group', id: groupid }, - }) - ) as GroupMembershipD[]; - }; - - const Plans = (proj: ProjectD) => - memory.cache.query((q) => - q.findRecords('plan').filter({ - relation: 'project', - record: { type: 'project', id: proj.id }, - }) - ) as PlanD[]; - - const Sections = (proj: ProjectD) => { - const plans = Plans(proj).map((pl) => pl.id); - const allsections = memory.cache.query((q) => - q.findRecords('section') - ) as SectionD[]; - return allsections.filter((s) => plans.includes(related(s, 'plan'))); - }; - - const Passages = (proj: ProjectD) => { - const sections = Sections(proj).map((s) => s.id); - const passages = memory.cache.query((q) => - q.findRecords('passage') - ) as PassageD[]; - return passages.filter((p) => sections.includes(related(p, 'section'))); - }; - - const SectionResources = (proj: ProjectD) => { - const sections = Sections(proj).map((s) => s.id); - return ( - memory.cache.query((q) => - q.findRecords('sectionresource') - ) as SectionResourceD[] - ).filter((r) => sections.includes(related(r, 'section'))); - }; - - const HighestByPassage = (media: MediaFileD[]) => { - const highest: MediaFileD[] = []; - let psg = ''; - media - .sort((a, b) => - related(a, 'passage') === related(b, 'passage') - ? a.attributes.versionNumber > b.attributes.versionNumber - ? 1 - : -1 - : related(a, 'passage') > related(b, 'passage') - ? 1 - : -1 - ) - .forEach((m) => { - if (related(m, 'passage') !== psg) { - highest.push(m); - psg = related(m, 'passage'); - } - }); - return highest; - }; - - const SourceMedia = (proj: ProjectD) => { - const sectionresourcemedia = SectionResources(proj).map( - (r) => related(r, 'mediafile') as string - ); - const media = memory.cache.query((q) => - q.findRecords('mediafile') - ) as MediaFileD[]; - const resourcemediafiles = media.filter((m) => - sectionresourcemedia.includes(m.id) - ); - const sourcemediafiles = media.filter( - (m) => - m.attributes?.readyToShare && - resourcemediafiles - .map((r) => related(r, 'resourcePassage')) - .includes(related(m, 'passage')) - ); - return HighestByPassage(sourcemediafiles); - }; - - const sharedNotePassageIds = (proj: ProjectD) => { - const psgs = Passages(proj).filter( - (p) => related(p, 'sharedResource') !== null - ); - const sharednotesids = psgs.map( - (p) => related(p, 'sharedResource') as string - ); - return ( - memory.cache.query((q) => - q.findRecords('sharedresource') - ) as SharedResourceD[] - ) - .filter((r) => sharednotesids.includes(r.id)) - .map((r) => related(r, 'passage') as string); - }; - - const sharedNotePassages = (proj: ProjectD) => { - const ids = sharedNotePassageIds(proj); - return ( - memory.cache.query((q) => q.findRecords('passage')) as PassageD[] - ).filter((p) => ids.includes(p.id)); - }; - - const sharedNoteSections = (proj: ProjectD) => { - const sectids = sharedNotePassages(proj).map((p) => related(p, 'section')); - return ( - memory.cache.query((q) => q.findRecords('section')) as SectionD[] - ).filter((s) => sectids.includes(s.id)); - }; - - const sharedNotePlans = (proj: ProjectD) => { - const planids = sharedNoteSections(proj).map((p) => related(p, 'plan')); - return (memory.cache.query((q) => q.findRecords('plan')) as PlanD[]).filter( - (s) => planids.includes(s.id) - ); - }; - - const sharedNoteProjects = (proj: ProjectD) => { - const projids = sharedNotePlans(proj).map((p) => related(p, 'project')); - return ( - memory.cache.query((q) => q.findRecords('project')) as ProjectD[] - ).filter((s) => projids.includes(s.id)); - }; - - const AllMediafiles = (proj: ProjectD) => { - const media = memory.cache.query((q) => - q.findRecords('mediafile') - ) as MediaFileD[]; - const plans = Plans(proj).map((pl) => pl.id); - const planmedia = media.filter((m) => plans.includes(related(m, 'plan'))); - const ips = IntellectualProperties(proj, needsRemoteIds).map((i) => - related(i, 'releaseMediafile') - ); - const ipmedia = media.filter((m) => ips.includes(m.id)); - const cats = ( - memory.cache.query((q) => - q.findRecords('artifactcategory') - ) as ArtifactCategoryD[] - ).filter( - (a) => - related(a, 'organization') === related(proj, 'organization') || - related(a, 'organization') === undefined - ); - const categorymediafiles = media.filter((m) => - cats.map((c) => related(c, 'titleMediafile') as string).includes(m.id) - ); - const orgkeytermtargets = OrgKeyTermTargets(proj, needsRemoteIds).map((i) => - related(i, 'mediafile') - ); - const okttmedia = media.filter((m) => orgkeytermtargets.includes(m.id)); - const sourcemediafiles = SourceMedia(proj); - const supportingNotePassages = sharedNotePassageIds(proj); - const sharedmedia = HighestByPassage( - media.filter((m) => - supportingNotePassages.includes(related(m, 'passage')) - ) - ); - const unique = new Set( - planmedia - .concat(ipmedia) - .concat(okttmedia) - .concat(categorymediafiles) - .concat(sourcemediafiles) - .concat(sharedmedia) - ); - return FromMedia(Array.from(unique), needsRemoteIds); - }; - - const FromPassages = ( - table: string, - proj: ProjectD | undefined, - remoteIds: boolean - ) => { - let recs = memory.cache.query((q) => q.findRecords(table)) as (BaseModel & - InitializedRecord)[]; - if (proj) { - const passages = Passages(proj).map((p) => p.id); - recs = recs.filter((rec) => passages.includes(related(rec, 'passage'))); - } - if (remoteIds) { - recs.forEach((r) => { - if (!remoteId(table, r.id, keyMap) && r.attributes) - r.attributes.offlineId = r.id; - if ( - table === 'mediafile' && - !remoteId('mediafile', related(r, 'sourceMedia'), keyMap) - ) { - (r as MediaFileD).attributes.sourceMediaOfflineId = related( - r, - 'sourceMedia' - ); - } - }); - } - return recs; - }; - - const Discussions = (proj: ProjectD | undefined, remoteIds: boolean) => { - let ds = memory.cache.query((q) => - q.findRecords('discussion') - ) as DiscussionD[]; - if (proj) { - const mediafiles = FromPassages('mediafile', proj, remoteIds).map( - (m) => m.id - ); - ds = ds.filter((rec) => mediafiles.includes(related(rec, 'mediafile'))); - } - if (remoteIds) { - ds.forEach((d) => { - if (!remoteId('discussion', d.id, keyMap) && d.attributes) - d.attributes.offlineId = d.id; - if (!remoteId('mediafile', related(d, 'mediafile'), keyMap)) - d.attributes.offlineMediafileId = related(d, 'mediafile'); - }); - } - return ds; - }; - - const OrgKeyTerms = (proj: ProjectD | undefined, remoteIds: boolean) => { - let kts = memory.cache.query((q) => - q.findRecords('orgkeyterm') - ) as OrgKeytermD[]; - kts = kts.filter( - (r) => - Boolean( - remoteId('organization', related(r, 'organization'), keyMap) - ) === needsRemoteIds - ); - if (proj) { - kts = kts.filter( - (rec) => related(rec, 'organization') === related(proj, 'organization') - ); - } - if (remoteIds) { - kts.forEach((kt) => { - if (!remoteId('orgkeyterm', kt.id, keyMap) && kt.attributes) - kt.attributes.offlineid = kt.id; - }); - } - return kts; - }; - - const OrgKeyTermTargets = ( - proj: ProjectD | undefined, - remoteIds: boolean - ) => { - let ktts = memory.cache.query((q) => - q.findRecords('orgkeytermtarget') - ) as OrgKeytermTargetD[]; - ktts = ktts.filter( - (r) => - Boolean( - remoteId('organization', related(r, 'organization'), keyMap) - ) === needsRemoteIds - ); - if (proj) { - ktts = ktts.filter( - (rec) => related(rec, 'organization') === related(proj, 'organization') - ); - } - if (remoteIds) { - ktts.forEach((ktt) => { - if (!remoteId('orgkeytermtarget', ktt.id, keyMap) && ktt.attributes) - ktt.attributes.offlineId = ktt.id; - if ( - related(ktt, 'mediafile') && - !remoteId('mediafile', related(ktt, 'mediafile'), keyMap) - ) - ktt.attributes.offlineMediafileId = related(ktt, 'mediafile'); - }); - } - return ktts; - }; - - const IntellectualProperties = ( - proj: ProjectD | undefined, - remoteIds: boolean - ) => { - let ips = memory.cache.query((q) => - q.findRecords('intellectualproperty') - ) as IntellectualPropertyD[]; - if (proj) { - ips = ips.filter( - (rec) => related(rec, 'organization') === related(proj, 'organization') - ); - } - if (remoteIds) { - ips.forEach((ip) => { - if (!remoteId('intellectualproperty', ip.id, keyMap)) - ip.attributes.offlineId = ip.id; - if (!remoteId('mediafile', related(ip, 'releaseMediafile'), keyMap)) - ip.attributes.offlineMediafileId = related(ip, 'releaseMediafile'); - }); - } - return ips; - }; - - const FromMedia = (media: MediaFileD[], remoteIds: boolean) => { - const copyMedia = media.map((m) => m); - if (remoteIds) { - copyMedia.forEach((m) => { - if (!remoteId('mediafile', m.id, keyMap) && m.attributes) { - m.attributes.offlineId = m.id; - } - const src = related(m, 'sourceMedia'); - if (src && !remoteId('mediafile', src, keyMap) && m.attributes) { - m.attributes.sourceMediaOfflineId = src; - } - delete m.attributes.planId; - delete m.attributes.artifactTypeId; - delete m.attributes.passageId; - delete m.attributes.userId; - delete m.attributes.recordedbyUserId; - delete m.attributes.recordedByUserId; - delete m.attributes.sourceMediaId; - }); - } - return copyMedia; - }; - - const Comments = (proj: ProjectD | undefined, remoteIds: boolean) => { - let comments = memory.cache.query((q) => - q.findRecords('comment') - ) as BaseModelD[]; - if (proj) { - const discussions = Discussions(proj, remoteIds); - const discussionIds = discussions.map((d) => d.id); - comments = comments.filter((rec) => - discussionIds.includes(related(rec, 'discussion')) - ); - } - if (remoteIds) { - comments.forEach((c) => { - if (!remoteId('comment', c.id, keyMap) && c.attributes) { - c.attributes.offlineId = c.id; - c.attributes.offlineDiscussionId = related(c, 'discussion'); - } - if ( - !remoteId('mediafile', related(c, 'mediafile'), keyMap) && - c.attributes - ) - c.attributes.offlineMediafileId = related(c, 'mediafile'); - }); - } - return comments; - }; - - const defaultQuery = (table: string) => - memory.cache.query((q) => q.findRecords(table)) as (BaseModel & - InitializedRecord)[]; - - const GetTableRecs = ( - info: fileInfo, - proj: ProjectD | undefined, - needsRemoteIds: boolean - ): BaseModelD[] => { - switch (info.table) { - case 'organization': - if (proj) - return [ - memory.cache.query((q) => - q.findRecord({ - type: 'organization', - id: related(proj, 'organization'), - }) - ) as OrganizationD, - ]; - return defaultQuery(info.table); - - case 'project': - if (proj) return [proj]; - return defaultQuery(info.table); - - case 'group': - if (proj) - return [ - memory.cache.query((q) => - q.findRecord({ - type: 'group', - id: related(proj, 'group'), - }) - ) as GroupD, - ]; - return defaultQuery(info.table); - - case 'groupmembership': - if (proj) return GroupMemberships(proj); - return defaultQuery(info.table); - - case 'user': - if (proj) { - const projusers = GroupMemberships(proj).map((gm) => - related(gm, 'user') - ); - const users = memory.cache.query((q) => - q.findRecords(info.table) - ) as UserD[]; - return users.filter( - (u) => projusers.find((p) => p === u.id) !== undefined - ); - } - return defaultQuery(info.table); - - case 'plan': - if (proj) return Plans(proj).concat(sharedNotePlans(proj)); - return defaultQuery(info.table); - - case 'section': - if (proj) return Sections(proj).concat(sharedNoteSections(proj)); - return defaultQuery(info.table); - - case 'passage': - if (proj) return Passages(proj).concat(sharedNotePassages(proj)); - return defaultQuery(info.table); - - case 'mediafile': - if (proj) return AllMediafiles(proj); - return FromMedia( - defaultQuery(info.table) as MediaFileD[], - needsRemoteIds - ); - - case 'passagestatechange': - return FromPassages(info.table, proj, needsRemoteIds); - - case 'discussion': - return Discussions(proj, needsRemoteIds); - - case 'comment': - return Comments(proj, needsRemoteIds); - - case 'projectintegration': - if (proj) - return memory.cache.query((q) => - q.findRecords(info.table).filter({ - relation: 'project', - record: { type: 'project', id: proj.id }, - }) - ) as BaseModelD[]; - return defaultQuery(info.table); - - case 'intellectualproperty': - return IntellectualProperties(proj, needsRemoteIds); - - case 'orgkeyterm': - return OrgKeyTerms(proj, needsRemoteIds); - - case 'orgkeytermtarget': - return OrgKeyTermTargets(proj, needsRemoteIds); - - default: - return defaultQuery(info.table).filter( - (r) => Boolean(r?.keys?.remoteId) === needsRemoteIds - ); - } - }; - const addJsonFile = ( table: string, recs: InitializedRecord[], sort: string ) => { - const resources = serializeRecords(recs, project, ser); + const resources = serializeRecords(recs, needsRemoteIds, ser); const json = '{"data":' + JSON.stringify(resources) + '}'; - const filename = `data/${sort}_${table}.json`; - files[filename] = json; + files[`data/${sort}_${table}.json`] = json; }; - for (const info of updateableFiles) { - const recs = GetTableRecs(info, project, needsRemoteIds); - if (recs?.length > 0) { - const filtered = needsRemoteIds - ? recs.filter((r) => Boolean(r?.keys?.remoteId)) - : recs; - if (filtered.length > 0) { - addJsonFile( - info.table + 's', - info.table === 'project' ? recs : filtered, - info.sort - ); - } - } - } + const addAll = ( + info: (typeof updateableFiles)[number], + excludeNew: boolean + ) => { + let recs = getTableRecs(info, project, needsRemoteIds); + if (!recs?.length) return; + if (needsRemoteIds && excludeNew) + recs = recs.filter((r) => Boolean(r.keys?.remoteId)); + if (recs.length > 0) addJsonFile(info.table + 's', recs, info.sort); + }; - for (const info of staticFiles) { - const recs = GetTableRecs(info, project, needsRemoteIds); - if (recs?.length > 0) { - addJsonFile(info.table + 's', recs, info.sort); - } - } + for (const info of updateableFiles) addAll(info, true); + for (const info of staticFiles) addAll(info, false); - const supportingProjects = sharedNoteProjects(project).filter((r) => + const projects = supportingProjects(project).filter((r) => Boolean(r.keys?.remoteId) ); - if (supportingProjects.length > 0) { - addJsonFile('supportingprojects', supportingProjects, 'Z'); - } + if (projects.length > 0) addJsonFile('supportingprojects', projects, 'Z'); + const orgs = supportingOrgs(project).filter((o) => Boolean(o.keys?.remoteId)); + if (orgs.length > 0) addJsonFile('supportingorgs', orgs, 'Z'); return files; } @@ -583,76 +92,37 @@ export function getOrganizationIntellectualPropertyFiles( organizationId: string ): OrganizationIntellectualPropertyExport { const ser = getSerializer(memory); - const keyMap = memory?.keyMap as RecordKeyMap; const orgs = memory.cache.query((q) => q.findRecords('organization') ) as OrganizationD[]; const org = orgs.find((o) => o.id === organizationId); const needsRemoteIds = Boolean(org?.keys?.remoteId); - - let ips = memory.cache.query((q) => - q.findRecords('intellectualproperty') - ) as IntellectualPropertyD[]; - ips = ips.filter((rec) => related(rec, 'organization') === organizationId); - - if (needsRemoteIds) { - ips.forEach((ip) => { - if (!remoteId('intellectualproperty', ip.id, keyMap) && ip.attributes) - ip.attributes.offlineId = ip.id; - if ( - !remoteId('mediafile', related(ip, 'releaseMediafile'), keyMap) && - ip.attributes - ) - ip.attributes.offlineMediafileId = related(ip, 'releaseMediafile'); - }); - } - - const releaseIds = ips - .map((ip) => related(ip, 'releaseMediafile')) - .filter((id): id is string => Boolean(id)); - const allMedia = memory.cache.query((q) => - q.findRecords('mediafile') + const { getTableRecs } = createExportCollector(memory, needsRemoteIds, { + organizationId, + }); + + const ips = getTableRecs( + { table: 'intellectualproperty', sort: 'I' }, + undefined, + needsRemoteIds + ); + const releaseMediafiles = getTableRecs( + { table: 'mediafile', sort: 'H' }, + undefined, + needsRemoteIds ) as MediaFileD[]; - let releaseMediafiles = allMedia.filter((m) => releaseIds.includes(m.id)); - releaseMediafiles = releaseMediafiles.map((m) => ({ ...m })); - if (needsRemoteIds) { - releaseMediafiles.forEach((m) => { - if (!remoteId('mediafile', m.id, keyMap) && m.attributes) { - m.attributes.offlineId = m.id; - } - const src = related(m, 'sourceMedia'); - if (src && !remoteId('mediafile', src, keyMap) && m.attributes) { - m.attributes.sourceMediaOfflineId = src; - } - delete m.attributes.planId; - delete m.attributes.artifactTypeId; - delete m.attributes.passageId; - delete m.attributes.userId; - delete m.attributes.recordedbyUserId; - delete m.attributes.recordedByUserId; - delete m.attributes.sourceMediaId; - }); - } - - const serializeForExport = (rec: InitializedRecord) => { - if (needsRemoteIds) { - return ser.serialize(rec) as unknown as Record; - } - const ri = ser.serialize(rec) as unknown as Record; - ri.id = rec.id; - ri.relationships = rec.relationships; - return ri; - }; const files: ProjectDataFiles = {}; if (ips.length > 0) { files['data/I_intellectualpropertys.json'] = - '{"data":' + JSON.stringify(ips.map((r) => serializeForExport(r))) + '}'; + '{"data":' + + JSON.stringify(serializeRecords(ips, needsRemoteIds, ser)) + + '}'; } if (releaseMediafiles.length > 0) { files['data/H_mediafiles.json'] = '{"data":' + - JSON.stringify(releaseMediafiles.map((r) => serializeForExport(r))) + + JSON.stringify(serializeRecords(releaseMediafiles, needsRemoteIds, ser)) + '}'; }