From b31b4f9dbe2d18d0c4df4993ba0c19d648e62ae5 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 22 Oct 2026 12:56:22 +0700 Subject: [PATCH] Python script to migrate non-SR projects Written by Claude Code (mostly Opus 5, some Opus 5.5), tested on local dev machine. Some projects, those with custom fields created in LF, will need a two-step process: run the script once, which will determine the XML to put into the FW project in order to define the custom fields. Then clone the Mercurial repo, add that XML as a new Mercurial commit, and push that commit to Lexbox. Then run the script again. The script is intended to run on the Language Forge `app` container, and that container cannot push Mercurial repos to LexBox, so the custom-fields commit needs to be a manual process. There are only six projects that look like they will need this, however, so it's not as onerous as it might sound. For all other projects, the script will run as-is without any manual steps. Once the script runs, there will be a `(projectCode)-migrate` project on the server with a copy of the project's data, and a new LexBox project with the same `(projectCode)-migrate` name that contains the project's data in FW format. That data can then be brought into FieldWorks Lite. --- docker-compose.yml | 3 + docker/app/Dockerfile | 6 + docker/lfmerge/Dockerfile | 2 +- scripts/copy_project_for_migration.py | 3300 +++++++++++++++++++++++++ 4 files changed, 3310 insertions(+), 1 deletion(-) create mode 100755 scripts/copy_project_for_migration.py diff --git a/docker-compose.yml b/docker-compose.yml index 35d8145697..33f2807515 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -91,6 +91,9 @@ services: # needed this volume mapping so changes to partials would be reflected in running app. - ./src/angular-app:/var/www/src/angular-app + # To be able to run Python migration scripts from inside the container in dev mode + - ./scripts:/var/www/scripts + lfmerge: build: context: docker/lfmerge diff --git a/docker/app/Dockerfile b/docker/app/Dockerfile index d803423066..f4c56674bb 100644 --- a/docker/app/Dockerfile +++ b/docker/app/Dockerfile @@ -55,6 +55,12 @@ COPY --from=sillsdev/web-languageforge:wait-latest /wait /wait # DEVELOPMENT BASE IMAGE FROM sillsdev/web-languageforge:base-php AS development-base RUN install-php-extensions xdebug-^3.1 +# Python for scripts/copy_project_for_migration.py, which runs in this container +# because it needs Mongo, the project assets volume and the LFMerge sendreceive +# volume. Development only; the production image has no need for it. +RUN apt-get update \ + && apt-get install -y --no-install-recommends python3 \ + && apt-get clean && rm -rf /var/lib/apt/lists/* COPY docker/app/docker-php-ext-xdebug.ini /usr/local/etc/php/conf.d RUN mv $PHP_INI_DIR/php.ini-development $PHP_INI_DIR/php.ini COPY --from=sillsdev/web-languageforge:wait-latest /wait /wait diff --git a/docker/lfmerge/Dockerfile b/docker/lfmerge/Dockerfile index 2e47bf37f7..00110f7700 100644 --- a/docker/lfmerge/Dockerfile +++ b/docker/lfmerge/Dockerfile @@ -1,2 +1,2 @@ -FROM ghcr.io/sillsdev/lfmerge:2.0.142 +FROM ghcr.io/sillsdev/lfmerge:2.0.143 # Do not add anything to this Dockerfile, it should stay empty diff --git a/scripts/copy_project_for_migration.py b/scripts/copy_project_for_migration.py new file mode 100755 index 0000000000..bc9dc8954b --- /dev/null +++ b/scripts/copy_project_for_migration.py @@ -0,0 +1,3300 @@ +#!/usr/bin/env python3 +"""Migrate a non-Send/Receive Language Forge project into the Send/Receive system. + +Given the projectCode of an existing project, this creates a complete copy whose +projectCode is the original with "-migrate" appended, points the copy's +sendReceiveProjectIdentifier at the LexBox-normalised form of that code, and +drives it through a full migration into LexBox. + +The original project is never modified. + +The source project must NOT already be a Send/Receive project: if it has a +non-empty sendReceiveProjectIdentifier the script exits with an error. Such a +project is already synced with a repository, and copying it would leave two LF +projects pointing into the same Send/Receive history. + +MIGRATION SEQUENCE + 1. Copy the sf_ database to sf_-migrate. + 2. Rewrite the copy's numeric option list keys as strings, and the two + numeric lexicon references that point at them. See OPTION LIST KEY + COERCION below. This touches the copy only. + 3. Give a fresh UUID4 to any lexicon entry in the copy whose guid is an + empty string. See EMPTY ENTRY GUIDS below. This touches the copy only. + 4. Stage the on-disk assets as real directories under the copy's own assets + folder. (They are converted to symlinks in step 8.) + 5. Insert the `projects` document: new _id, new projectCode, and + sendReceiveProjectIdentifier set to the normalised form of that code + (see send_receive_identifier -- underscores become hyphens and CamelCase + is split, because LexBox project codes must be lowercase). + 6. init_lexbox_project() logs in to LexBox (POST /api/login), calls + POST /api/project/initFwDataProject with the writing systems taken from + the LF project config. A 200 from that endpoint is taken as confirmation + that the project exists. Every writing system the project knows about + must be covered by the vernacular or analysis list; any that is not is + warned about and sent as analysis. + 7. Remove any webwork clone and LFMerge state file an earlier attempt left + behind (see clear_send_receive_state -- the LexBox project just created + has a new root changeset, so an older clone is unrelated to it), then run + the first Send/Receive. LFMerge clones the repository into + webwork// and pushes the Mongo data (lexicon, comments, + option lists) up to LexBox. Then wait for it to finish. + 8. Move any asset not already present in + webwork//LinkedFiles/{AudioVisual,Pictures} into that tree, + and replace the copy's assets//sf_/{audio,pictures} + directories with symlinks pointing there -- the layout a real + Send/Receive project has (LexProjectModel::createAssetsFolders). + 9. Second Send/Receive, to push those assets up to LexBox. + + Steps 7 and 9 wait on the LFMerge state file, + sendreceive/state/.state, so the phases stay ordered. + +WHERE TO RUN THIS + Inside the `app` container, which is where the Mongo host, the project assets + volume and the LFMerge sendreceive volume are all reachable: + + docker compose exec app python3 /var/www/scripts/copy_project_for_migration.py + + The base image (php:7.4-apache) ships no Python, so the development-base + stage of docker/app/Dockerfile installs python3. If `python3: not found`, + rebuild the image with `make build`. + + No pip packages are needed. This script deliberately uses only the standard + library plus the mongosh/mongodump/mongorestore binaries that the app image + already installs (see docker/base-php/Dockerfile). + + Note that the app and lfmerge containers share both the assets volume and the + sendreceive volume (docker-compose.yml), so the files this script writes are + the same files LFMerge reads. + +WHAT GETS COPIED + * the `projects` document in the main database + * the project's own database, sf_, with all collections and + indexes (lexicon, lexiconComments, optionlists, activity, messages) + * the on-disk assets, assets//sf_/{audio,pictures} + * membership: the new project's _id is added to each member's + users.projects array + +OPTION LIST KEY COERCION + Option list item keys are meant to be strings, but some projects store them + as BSON int32: in the 2026-07-06 corpus, 39,759 such keys across 53 projects, + nearly all in the `anthro-code` list (reached from config as + senses.anthropologyCategories), the rest in `domain-type` and + `pers-feature-value`. Step 2 rewrites every one of them in the copy as the + string of that integer, which is exactly what each item's `abbreviation` + already holds. Coercion cannot collide: no coerced key equals an existing + string key in the same list, and no two integers coerce to the same string. + + The lexicon almost always already references these items by string ("27", + "130"), so coercing the lists is what brings the two sides into agreement. + Two senses in the whole corpus are the exception -- they reference by + integer, and would be left dangling by a list-only coercion, so they are + converted too. See SPECIAL_LEXICON_INT_REFERENCES; those two senses are the + only lexicon documents this script modifies for the sake of option lists. + The other lexicon change it makes is described next. + +EMPTY ENTRY GUIDS + Five projects in the 2026-07-06 corpus have exactly one lexicon entry whose + guid is an empty string. That is the whole of the problem: no nulls, no + missing guid fields, no malformed values, and entries only -- senses and + examples never exhibit it. Step 3 therefore matches `guid: ""` exactly and + leaves every other guid alone, including one that is a valid UUID of some + version other than 4, which LfMerge parses happily. + + An empty guid does not make LfMerge fail, which is why it has gone + unnoticed. LfMerge reads the field with Guid.TryParse + (MongoConnection.CanParseGuid), so the entry is quietly dropped from the + ObjectId->Guid map that comments are attached by, and + ConvertMongoToLcmLexicon then falls back to `lfEntry.Guid ?? Guid.Empty`. + The entry would be created in FieldWorks under the all-zeros GUID, and a + second entry in the same state would find that object and overwrite it + rather than create its own -- so entries would silently merge instead of + erroring. + +WHAT IS DELIBERATELY NOT TOUCHED + * The SOURCE project, in every respect. All coercion happens in the copy, + after the database has been duplicated. + * Every other lexicon value. The only lexicon writes are the two senses + named in SPECIAL_LEXICON_INT_REFERENCES and the empty entry guids of + step 3; any other numeric option list reference is reported as a warning + and left exactly as it is. + * Guids on senses and examples. They are nullable in the same way, but the + empty-string problem does not arise there, so they are not scanned. + * `abbreviation` and `value` on option list items, which are already strings. + * Sync state is copied verbatim: lastSyncedDate and the per-entry dirtySR + flags are left exactly as the original has them. Because the source has + never synced, that means lastSyncedDate is still its default of epoch 0 -- + which is what you want for a first push into an empty repository. + * `sendReceiveProject` ({name, repository, role}) is not populated. The + source has no Send/Receive configuration to copy, so the copy ends up with + sendReceiveProjectIdentifier set but no repository details. LFMerge does + not need them -- it works from the identifier -- but the LF UI displays + them, so init_lexbox_project() is the place to record them if wanted. + * `userrelation` unread-activity documents are not copied. +""" + +from __future__ import annotations + +import argparse +import http.cookiejar +import json +import os +import re +import shutil +import ssl +import subprocess +import sys +import tempfile +import time +import urllib.error +import urllib.request +import uuid +from typing import Any, Callable +from urllib.parse import quote_plus, urlencode, urlsplit, urlunsplit +from xml.sax.saxutils import escape as xml_escape + +MIGRATE_SUFFIX = "-migrate" + +# Defaults match the `app` service environment in docker-compose.yml, and the +# hardcoded LFMerge paths in SendReceiveCommands.php. +DEFAULT_MONGODB_CONN = "mongodb://lf-db:27017" +DEFAULT_DATABASE = "scriptureforge" +DEFAULT_ASSETS_ROOT = "/var/www/html/assets" +DEFAULT_SENDRECEIVE_BASE = "/var/lib/languageforge/lexicon/sendreceive" + +# (assets subfolder, LinkedFiles subfolder) -- LexProjectModel::createAssetsFolders +ASSET_LINKED_FILES = (("audio", "AudioVisual"), ("pictures", "Pictures")) + +# The only two lexicon values in the whole corpus that reference an option list +# item by a NUMBER rather than by a string. Everywhere else the lexicon already +# uses strings ("27", "130", "824.1"), so coercing the option lists on its own +# lines the two sides up -- but these two would be left pointing at a key that +# no longer exists, so they are coerced with them. +# +# Keyed by SOURCE projectCode (the copy's code is that plus --suffix, so the +# source is what identifies the data). The value is the field to look in and the +# exact `values` list that must be present, so a sense is only rewritten when +# both the project and the list match. Anything else numeric is reported and +# left alone. Found by scanning the 2026-07-06 mongodump: +# +# test-project-d entry 58bfb4f78b2147101c74ece2 [18] +# vocabularium_totius_latinitatis entry 61de9ccb499649193b5d5ce5 [104] +# +# Both are anthropologyCategories, whose list is anthro-code. Both projects are +# tiny (65 and 3 entries), which is why nobody hit this before. +SPECIAL_LEXICON_INT_REFERENCES: dict[str, tuple[str, list[int]]] = { + "test-project-d": ("anthropologyCategories", [18]), + "vocabularium_totius_latinitatis": ("anthropologyCategories", [104]), +} + +# SRState values, from src/angular-app/bellows/shared/model/send-receive-state.model.ts +SR_IN_PROGRESS = frozenset({"CLONING", "SYNCING", "PENDING", "LF_CLONING"}) +SR_DONE = frozenset({"IDLE", "SYNCED"}) +SR_FAILED = frozenset({"HOLD", "ERROR"}) + +WEB_USER = "www-data" + +# LexBox. The JWT is delivered as this cookie, whose value literally IS the JWT +# (AuthKernel.cs:229 overrides the cookie ticket format to write a JWT). +# In development LexBox runs outside this Docker Compose environment, reached +# through the "admin.localhost:host-gateway" extra_hosts entry that the app and +# lfmerge services already declare, over plain HTTP. Production is lexbox.org. +DEFAULT_LEXBOX_HOST = "http://admin.localhost" +DEFAULT_LEXBOX_USER = "admin" +DEFAULT_LEXBOX_PASSWORD = "pass" +LEXBOX_AUTH_COOKIE = ".LexBoxAuth" +# Project.cs:12 -- LexBox rejects codes that do not match, and codes under 4 chars. +LEXBOX_CODE_RE = re.compile(r"^[a-z\d][a-z\-\d]*$") +LEXBOX_CODE_MIN_LENGTH = 4 +# liblcm seeds a new project's anthropology list (the OCM outline) from a file shipped with +# SIL.LCModel and writes each label into the project's DEFAULT ANALYSIS writing system -- which is +# simply the first entry of wsAnalysis. The OCM file has no labels for an unlisted-language ("qaa") +# writing system, so one of those in first position makes CreateNewLangProj die inside +# InitializeAnthroList with +# System.ArgumentOutOfRangeException ... (Parameter 'ws') +# and the project is never created. Every OCM entry does have an `en` label, so when the list leads +# with a qaa tag, `en` is moved (or added) to the front and the rest keeps its order behind it. +# Any other leading tag is left alone: reordering a working project's writing systems for no reason +# would change which one FieldWorks treats as its default analysis writing system. +ANTHRO_DEFAULT_ANALYSIS_WS = "en" +UNLISTED_LANGUAGE_PREFIX = "qaa" + +# A writing system declared in both the vernacular and the analysis list gets taken out of one of +# them, so that each one has a single role in FieldWorks. Which one it leaves is decided by where +# its data actually is, not by which field the config happens to declare it in -- the config +# declaration can rest on a single stray value. One project declares "pt" on its lexeme field +# because one entry of 1248 has the vernacular headword duplicated into its "pt" alternative, while "pt" +# holds 1239 glosses, 1121 definitions and 1096 example translations. Going by the declaration would +# have made Portuguese vernacular-only and left the project with "en", which has three values in it. +# +# This writing system is never removed from the ANALYSIS list, whichever way its data leans. It is +# "en" for the same reason ANTHRO_DEFAULT_ANALYSIS_WS is: liblcm writes the anthropology list in the +# default analysis writing system and "en" is the only one the OCM file is guaranteed to have labels +# for, so a project left with no "en" in analysis can fail to be created at all. It can still be +# removed from the vernacular list, which is the usual outcome when a project glosses in English. +VERNACULAR_ANALYSIS_OVERLAP_EXEMPTION = "en" + +# Custom fields. LF names them by the level they live at -- LfMerge's MagicStrings has +# customField_entry / customField_senses / customField_example -- and the config nests them under +# entry.fields, senses.fields and senses.fields.examples.fields to match. +CUSTOM_FIELD_PREFIX = "customField_" + +# What a future LexBox API would need to create the field in FieldWorks. This is the inverse of +# LfMerge's CellarPropertyTypeToLfCustomFieldType (ConvertLcmToMongoCustomField.cs). That map is not +# injective -- MultiString and MultiUnicode both come out as multitext -- so the inverse picks +# MultiUnicode, which is what LF round-trips. +LCM_TYPE_FOR_LF_TYPE = { + "multitext": "MultiUnicode", + "optionlist": "ReferenceAtomic", + "multioptionlist": "ReferenceCollection", + "multiparagraph": "OwningAtomic", +} + +# Shapes LF writes into the lexicon, used to guess the type of a field the config never declared. +# A guess is not a declaration, so anything derived this way is reported as one. +# shape -> (LF type, LCM type) +LF_TYPE_FOR_VALUE_SHAPE = { + "multitext": ("multitext", "MultiUnicode"), + "single-listref": ("optionlist", "ReferenceAtomic"), + "multi-listref": ("multioptionlist", "ReferenceCollection"), + "paragraphs": ("multiparagraph", "OwningAtomic"), + # LF has no Date or Number field type; it stores those as a multitext under a fake writing + # system (LfMerge's MagicStrings.LanguageCodeForGenDateFields / ...ForIntFields). No project in + # the migration backlog uses either, but the shape is recognised so one would not be mistaken + # for text. + "date": ("date (fake ws qaa-Qaad)", "GenDate"), + "number": ("number (fake ws qaa-Zmth)", "Integer"), +} + +# Option lists FieldWorks already has, from LfMerge's MagicStrings.LcmOptionlistNames. A custom +# field pointing at anything else needs its possibility list created before the field can be. +STANDARD_LF_OPTION_LIST_CODES = frozenset({ + "grammatical-info", "semantic-domain-ddp4", "domain-type", "environments", "location", + "usage-type", "reversal-type", "sense-type", "anthro-code", "status", "etymology", + "lexical-relation", "note-type", "paradigm", "users", "translation-type", + "from-part-of-speech", "morph-type", "noun-slot", "verb-slot", "stative-slot", + "noun-infl-class", "verb-infl-class", +}) + +# Custom field names that FieldWorks may or may not treat as the same field. FLEx is inconsistent +# about two names differing only by case -- sometimes one field, sometimes two -- so the migration +# gives one of them a distinct name before it ever reaches FieldWorks. +# +# One project has "Class Marker" and "Class marker" at the entry level, and they are NOT +# duplicates. 830 entries fill only the first, 1279 only the second, and the one entry that fills +# both holds the same value in each, so nothing is lost either way -- but they hold different +# conventions. "Class Marker" is the older field, populated in 2014, and every one of its 832 +# values is a noun class morpheme ("mu", "ki") or the "##" placeholder, never a class number; +# "Class marker" is the 2016 field and mixes 1108 morphemes with 172 class numbers. Renaming rather +# than merging keeps the two apart so the distinction stays recoverable. +# +# Another project has "Morphophonemic Prop" and "MorphoPhonemic Prop" at the sense level, and here +# the two are not comparable at all. The lowercase-p one holds real data: 167 senses, all in "en", +# carrying a controlled vocabulary of vernacular morphophonemic properties ("underlong", "PMlowers", +# "RECIP; PMlowers; Mlowers; foreshortens") across 141 distinct values, nearly all written in 2014. +# The capital-P one has three senses, all in "es", holding ".", "x" and "do not load for CADA" -- +# a scratch field somebody made by mistyping the capitalisation, used as a marker and abandoned. +# So it is renamed out of the way rather than merged, which would fold three junk values into a +# clean linguistic field. No sense fills both, and the two use different writing systems, so a +# merge would in fact have been lossless; it just would not have been sensible. +# +# Keyed by field NAME rather than by project, since the project will not necessarily carry the same +# code or name in production as it does in the test snapshot. Nothing here is level-specific: a +# name is renamed wherever it clashes, and "Class Marker" happens to be an entry field while +# "MorphoPhonemic Prop" is a sense field. +CASE_CLASH_CUSTOM_FIELD_RENAMES = { + "Class Marker": "Class Marker Morpheme", + "MorphoPhonemic Prop": "MorphoPhonemic Prop Unused", +} + +# The LCM class each level's fields belong to, for the same future API. +LCM_CLASS_FOR_LEVEL = {"entry": "LexEntry", "senses": "LexSense", "examples": "LexExampleSentence"} + +# --- FLExProject.CustomProperties ------------------------------------------ # +# FieldWorks keeps its custom field definitions in FLExProject.CustomProperties, the one file in +# revision 0 of a FLEx Mercurial repository. LfMerge will not create them itself -- it logs +# "Custom field X from LF skipped, because we're not yet creating new custom fields in LCM" -- so +# the definitions have to be in the repository BEFORE the first Send/Receive, which is the only +# pass that would have carried the data. That is why a project with custom fields stops short of +# Send/Receive and prints this file for a human to commit. +# +# The format is copied from real Chorus repositories, not invented: CRLF line endings, tab +# indentation, one attribute per line in alphabetical order, elements sorted by class then name, +# and no trailing newline after . +CUSTOM_PROPERTIES_EOL = "\r\n" +CUSTOM_PROPERTIES_HEADER = '' +CUSTOM_PROPERTIES_FILENAME = "FLExProject.CustomProperties" + +# FieldWorks "magic" writing system selectors, all four of which occur here. The choice is only +# about which writing systems FLEx offers in the field; it does not change the field's type, which +# is always MultiUnicode because LF never had a single-string custom field. +# +# -1 and -2 name one writing system each, and are worth reaching for: a great many custom fields +# hold data in exactly one writing system, and a field showing one row rather than one row per +# writing system in a list is what that data actually is. +# +# The combined selectors matter because a field is not obliged to stay on one side of the project's +# vernacular/analysis divide. One project's "Top 10 list" holds Thai in 314 entries and English +# in 223; -4 alone would leave the English out of the field entirely. +WS_SELECTOR_FIRST_ANALYSIS = -1 +WS_SELECTOR_FIRST_VERNACULAR = -2 +WS_SELECTOR_ALL_ANALYSIS = -3 +WS_SELECTOR_ALL_VERNACULAR = -4 +WS_SELECTOR_ANALYSIS_THEN_VERNACULAR = -5 +WS_SELECTOR_VERNACULAR_THEN_ANALYSIS = -6 + +# destclass for the types that need one: 7 is CmPossibility, what a list reference points at, and +# 14 is StText, what a multiparagraph owns. +LCM_DEST_CLASS_FOR_TYPE = { + "ReferenceAtomic": 7, + "ReferenceCollection": 7, + "OwningAtomic": 14, +} + +# listRoot GUIDs of the possibility lists in a project FwHeadless creates, read from the +# NewLangProj.fwdata template it builds every project from. These are NOT universal -- a project +# made by an older FieldWorks has different GUIDs for the same lists, which is why they come from +# the template and not from some existing project. +# +# Only lists that are BOTH in the template and mapped to an LF list code by LfMerge +# (ConvertLcmToMongoCustomField.GuidToListCode) are here. grammatical-info and +# semantic-domain-ddp4 are mapped by LfMerge but PartsOfSpeech and SemanticDomainList are absent +# from the template, so a fresh project has no GUID to point at; any other list arrives as its +# NAME rather than a code, and has to be created by hand first. +LCM_LIST_ROOT_GUIDS = { + "anthro-code": "d87ba7c6-ea5e-11de-9dd2-0013722f8dec", + "domain-type": "b0a1eb98-ea5e-11de-888e-0013722f8dec", + "location": "d887937e-ea5e-11de-985e-0013722f8dec", + "sense-type": "af32c340-ea5e-11de-9040-0013722f8dec", + "status": "d86632ba-ea5e-11de-8a58-0013722f8dec", + "usage-type": "af542404-ea5e-11de-8bde-0013722f8dec", +} +LIST_ROOT_PLACEHOLDER = "FILL-IN-LIST-ROOT-GUID" + +# initFwDataProject builds a template .fwdata and pushes it; the LexBox test +# allows 5 minutes and disables retries because it is NOT idempotent. +DEFAULT_LEXBOX_TIMEOUT_SECONDS = 300 + +# How long to wait for lfmergeqm to dequeue our request before giving up, and +# how long a whole Send/Receive may take. +PICKUP_TIMEOUT_SECONDS = 90 +DEFAULT_SR_TIMEOUT_SECONDS = 1800 +POLL_INTERVAL_SECONDS = 2.0 + + +# --------------------------------------------------------------------------- # +# configuration +# --------------------------------------------------------------------------- # + + +class Config: + """Connection and path settings, read from the environment.""" + + def __init__(self) -> None: + self.conn = os.environ.get("MONGODB_CONN", DEFAULT_MONGODB_CONN) + self.user = os.environ.get("MONGODB_USER", "") + self.password = os.environ.get("MONGODB_PASS", "") + self.authsource = os.environ.get("MONGODB_AUTHSOURCE", "admin") + self.database = os.environ.get("DATABASE", DEFAULT_DATABASE) + self.assets_root = os.environ.get("ASSETS_ROOT", DEFAULT_ASSETS_ROOT) + self.sendreceive_base = os.environ.get("LFMERGE_BASE_DIR", DEFAULT_SENDRECEIVE_BASE) + + @property + def syncqueue_path(self) -> str: + return os.path.join(self.sendreceive_base, "syncqueue") + + def queue_file(self, project_code: str) -> str: + # LfMerge's Queue.cs reads the FILENAME as the project code. + return os.path.join(self.syncqueue_path, project_code) + + def state_file(self, project_code: str) -> str: + # SendReceiveCommands.php:213 lowercases the code for the state file. + return os.path.join(self.sendreceive_base, "state", project_code.lower() + ".state") + + def webwork_path(self, project_code: str) -> str: + # LexProjectModel::getSendReceiveWorkFolder() lowercases the code. + return os.path.join(self.sendreceive_base, "webwork", project_code.lower()) + + def linked_files_path(self, project_code: str) -> str: + return os.path.join(self.webwork_path(project_code), "LinkedFiles") + + def assets_path(self, app_name: str, db_name: str) -> str: + return os.path.join(self.assets_root, app_name, db_name) + + def uri(self, database: str | None = None) -> str: + """Build a mongodb:// URI with credentials embedded. + + The Mongo CLI tools refuse to combine --uri with --username/--db, so + everything has to go into the URI itself. + """ + parts = urlsplit(self.conn) + netloc = parts.netloc + if "@" in netloc: # credentials already supplied in MONGODB_CONN + host = netloc + else: + host = netloc + if self.user: + host = f"{quote_plus(self.user)}:{quote_plus(self.password)}@{netloc}" + # The leading "/" is required before any query string, so always emit it. + path = "/" + (database or "") + query = parts.query + if self.authsource and "authSource" not in query: + query = f"{query}&authSource={self.authsource}" if query else f"authSource={self.authsource}" + return urlunsplit((parts.scheme or "mongodb", host, path, query, "")) + + def safe_uri(self, database: str | None = None) -> str: + """The URI with the password masked, for logging.""" + uri = self.uri(database) + if self.password: + uri = uri.replace(quote_plus(self.password), "****") + return uri + + +# --------------------------------------------------------------------------- # +# helpers +# --------------------------------------------------------------------------- # + + +def log(message: str) -> None: + print(message, file=sys.stderr, flush=True) + + +class Fatal(Exception): + """An error that should abort the run (and trigger rollback).""" + + +def database_name(project_code: str) -> str: + """Replicate ProjectModel::databaseName() exactly (ProjectModel.php:115).""" + return "sf_" + project_code.lower().replace(" ", "_") + + +def send_receive_identifier(project_code: str) -> str: + """Derive a Send/Receive (LexBox) project code from a projectCode. + + Underscores become hyphens. An uppercase letter immediately following a + lowercase letter becomes a hyphen plus its lowercase form, which splits + CamelCase into words. Every other uppercase letter simply becomes lowercase, + so a run of capitals is not broken up. + + TitleCaseProject -> title-case-project + FOO_BAR -> foo-bar + + LexBox project codes must match ^[a-z\\d][a-z-\\d]*$, so this makes the + common Language Forge naming styles usable. It does not fix everything -- + spaces and other punctuation survive and are caught by validation. + """ + result = [] + for index, character in enumerate(project_code): + if character == "_": + result.append("-") + elif character.isupper(): + # "immediately follows a lowercase letter" is judged against the + # original string, before underscores were rewritten. + if index > 0 and project_code[index - 1].islower(): + result.append("-") + result.append(character.lower()) + else: + result.append(character) + return "".join(result) + + +def require_tool(name: str) -> None: + if shutil.which(name) is None: + raise Fatal( + f"Required tool '{name}' was not found on PATH. " + "This script is meant to run inside the app container, which provides " + "mongosh and mongodb-org-tools." + ) + + +def run(cmd: list[str], *, what: str) -> str: + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + raise Fatal(f"{what} failed (exit {result.returncode}):\n{result.stderr.strip()}") + return result.stdout + + +def mongosh_json(config: Config, js: str, *, what: str) -> Any: + """Run a mongosh script and parse the single JSON value it prints. + + The script is passed via a temp file so there is no shell quoting to get + wrong, and values are interpolated with json.dumps. + """ + require_tool("mongosh") + with tempfile.NamedTemporaryFile("w", suffix=".js", delete=False) as handle: + handle.write(js) + script_path = handle.name + try: + output = run(["mongosh", config.uri(), "--quiet", "--file", script_path], what=what) + finally: + os.unlink(script_path) + + for line in reversed(output.splitlines()): + line = line.strip() + if line: + try: + return json.loads(line) + except json.JSONDecodeError as exc: + raise Fatal(f"{what}: could not parse mongosh output {line!r}: {exc}") from exc + raise Fatal(f"{what}: mongosh produced no output") + + +def chown_to_web_user(path: str, *, recursive: bool = True) -> None: + """Best-effort chown to www-data, so LFMerge and Apache can use the files. + + LFMerge runs lfmergeqm as www-data, so anything this script drops into the + sendreceive tree has to be readable and writable by that user. + """ + try: + import grp + import pwd + + uid = pwd.getpwnam(WEB_USER).pw_uid + gid = grp.getgrnam(WEB_USER).gr_gid + except (ImportError, KeyError): + return + + def attempt(target: str) -> None: + try: + os.chown(target, uid, gid, follow_symlinks=False) + except (PermissionError, FileNotFoundError, NotImplementedError, TypeError): + try: + os.chown(target, uid, gid) + except (PermissionError, FileNotFoundError): + pass + + attempt(path) + if recursive and os.path.isdir(path) and not os.path.islink(path): + for root, dirnames, filenames in os.walk(path): + for name in dirnames + filenames: + attempt(os.path.join(root, name)) + + +# --------------------------------------------------------------------------- # +# mongo steps +# --------------------------------------------------------------------------- # + + +def fetch_project(config: Config, project_code: str) -> dict | None: + js = f""" +const main = db.getSiblingDB({json.dumps(config.database)}); +const doc = main.projects.findOne({{ projectCode: {json.dumps(project_code)} }}); +print(EJSON.stringify(doc === null ? {{ found: false }} : {{ + found: true, + id: doc._id.toString(), + projectCode: doc.projectCode, + projectName: doc.projectName || "", + appName: doc.appName || "", + siteName: doc.siteName || "", + isArchived: doc.isArchived === true, + sendReceiveProjectIdentifier: doc.sendReceiveProjectIdentifier || "", + memberCount: Object.keys(doc.users || {{}}).length, +}})); +""" + result = mongosh_json(config, js, what="Reading project record") + return result if result.get("found") else None + + +def dedup(tags: list[str]) -> list[str]: + """Drop duplicates and blanks, preserving first-seen order.""" + seen: set[str] = set() + result = [] + for tag in tags: + tag = (tag or "").strip() + if tag and tag not in seen: + seen.add(tag) + result.append(tag) + return result + + +def fetch_writing_systems(config: Config, project_code: str) -> dict[str, list[str]]: + """Read the ordered vernacular and analysis writing system tags from the config. + + Language Forge has no vernacular/analysis flag anywhere -- the distinction is + conventional, encoded in which config field a tag appears under: + + vernacular: config.entry.fields.lexeme.inputSystems (index 0 is the + project's primary language, per LexProjectModel.php:38) + plus config.entry.fields.citationForm.inputSystems + analysis: config.entry.fields.senses.fields.definition.inputSystems + plus ...senses.fields.gloss.inputSystems + + The lists are ordered BSON arrays of hyphenated BCP-47 tags and are passed + through verbatim. The secondary fields are unioned in because a project may + populate gloss but not definition (or citationForm but not lexeme), and a + writing system missing from the FieldWorks project could not receive data. + Order is preserved, primary field first. + + Also returns the complete set of writing systems the project knows about, so + the caller can check that vernacular + analysis covers all of them: + + config_all: every inputSystems entry anywhere under config.entry, found + by walking the whole field tree (fields can nest, e.g. + senses.examples.sentence). Ordered by fieldOrder where the + config provides one, so the order is the config's own. + declared: the keys of project.inputSystems, which is the project's map + of writing system definitions (InputSystem.php). This is a + BSON document, so its order is insertion order and is not + meaningful -- it is only used to catch writing systems that + no config field references. + """ + js = f""" +const main = db.getSiblingDB({json.dumps(config.database)}); +const doc = main.projects.findOne( + {{ projectCode: {json.dumps(project_code)} }}, + {{ config: 1, inputSystems: 1 }} +); +const entry = (((doc || {{}}).config || {{}}).entry) || {{}}; +const fields = entry.fields || {{}}; +const senses = ((fields.senses || {{}}).fields) || {{}}; + +function tags(field) {{ + return (field && Array.isArray(field.inputSystems)) ? field.inputSystems : []; +}} + +// Walk the entire config field tree, collecting every inputSystems entry. +// fieldOrder is the order the user sees, so honour it and append any field not +// listed there; nested field lists (senses, examples) are recursed into. +const configAll = []; +function walk(fieldList) {{ + if (!fieldList || typeof fieldList !== "object") return; + const children = fieldList.fields || {{}}; + const order = Array.isArray(fieldList.fieldOrder) ? fieldList.fieldOrder : []; + const names = []; + order.forEach(function (name) {{ + if (children[name] && names.indexOf(name) === -1) names.push(name); + }}); + Object.keys(children).forEach(function (name) {{ + if (names.indexOf(name) === -1) names.push(name); + }}); + names.forEach(function (name) {{ + const child = children[name]; + tags(child).forEach(function (tag) {{ configAll.push(tag); }}); + if (child && child.fields) walk(child); + }}); +}} +walk(entry); + +print(EJSON.stringify({{ + lexeme: tags(fields.lexeme), + citationForm: tags(fields.citationForm), + definition: tags(senses.definition), + gloss: tags(senses.gloss), + configAll: configAll, + declared: Object.keys((doc || {{}}).inputSystems || {{}}), +}})); +""" + found = mongosh_json(config, js, what="Reading writing systems from project config") + return { + "lexeme": list(found["lexeme"]), + "citationForm": list(found["citationForm"]), + "definition": list(found["definition"]), + "gloss": list(found["gloss"]), + "config_all": list(found["configAll"]), + "declared": list(found["declared"]), + } + + +def fetch_custom_fields(config: Config, project_code: str) -> dict[str, dict]: + """The custom fields the project config declares, keyed by LF field name. + + Each value carries the level the field lives at, the LF `type` discriminator, the label, the + writing systems the field uses and, for list-backed fields, the option list code. + + The config is authoritative for a field's TYPE, but not for whether it is used: 8 fields in the + 2026-07-06 corpus are declared and never populated, and 32 are populated without being declared. + scan_custom_field_usage() supplies the other half. + """ + js = f""" +const mainDb = {json.dumps(config.database)}; +const projectCode = {json.dumps(project_code)}; +const prefix = {json.dumps(CUSTOM_FIELD_PREFIX)}; +""" + """ +const main = db.getSiblingDB(mainDb); +const doc = main.projects.findOne({ projectCode: projectCode }, { config: 1 }); +const entry = (((doc || {}).config || {}).entry) || {}; + +const found = {}; + +// The three levels are spelled out rather than walked generically, so that the level recorded for +// each field is exactly the one its name is prefixed with. +function collect(fieldList, level) { + const fields = (fieldList || {}).fields || {}; + Object.keys(fields).forEach(function (name) { + if (name.indexOf(prefix) !== 0) return; + const field = fields[name] || {}; + found[name] = { + level: level, + type: field.type || null, + label: field.label || null, + inputSystems: Array.isArray(field.inputSystems) ? field.inputSystems : [], + listCode: field.listCode || null + }; + }); + return fields; +} + +const entryFields = collect(entry, "entry"); +const senseFields = collect(entryFields.senses, "senses"); +collect(senseFields.examples, "examples"); + +print(EJSON.stringify(found)); +""" + return mongosh_json(config, js, what=f"Reading custom field config for {project_code}") + + +def scan_custom_field_usage(config: Config, db_name: str) -> dict[str, dict]: + """How each custom field is actually used in db_name.lexicon, keyed by LF field name. + + Reports, per field: the level it was seen at, how many entries carry a value, how many values + in total, the value shapes observed (so a field the config does not declare can still be typed), + the writing systems its values use, and how many entries carry data in each of them. + + The per-writing-system counts are what a human uses to check a guess: a field whose data is + almost all in the vernacular writing system is a vernacular field whatever the config says. + + Shape detection mirrors what LF writes: a multitext is {ws: {value: ...}}, a single option list + reference is {value: ...}, a multiple one is {values: [...]}. + """ + js = f""" +const targetName = {json.dumps(db_name)}; +""" + """ +const target = db.getSiblingDB(targetName); +const found = {}; + +function isEmpty(value) { + if (value === null || value === undefined) return true; + if (Array.isArray(value)) return value.length === 0; + if (typeof value === "string") return value === ""; + if (typeof value === "object") { + if (Object.prototype.hasOwnProperty.call(value, "values")) return (value.values || []).length === 0; + if (Object.prototype.hasOwnProperty.call(value, "value")) return value.value === null || value.value === ""; + return Object.keys(value).every(function (k) { return isEmpty(value[k]); }); + } + return false; +} + +function shapeOf(value) { + if (value === null || typeof value !== "object") return "scalar"; + if (Array.isArray(value)) return "array"; + // Order matters: a multitext has no "value"/"values" key of its own, only writing systems. + if (Object.prototype.hasOwnProperty.call(value, "values")) return "multi-listref"; + if (Object.prototype.hasOwnProperty.call(value, "value")) return "single-listref"; + if (Object.prototype.hasOwnProperty.call(value, "paragraphs")) return "paragraphs"; + const keys = Object.keys(value); + if (keys.indexOf("qaa-Qaad") !== -1) return "date"; + if (keys.indexOf("qaa-Zmth") !== -1) return "number"; + const allMultitext = keys.length > 0 && keys.every(function (k) { + const inner = value[k]; + return inner && typeof inner === "object" && Object.prototype.hasOwnProperty.call(inner, "value"); + }); + return allMultitext ? "multitext" : "unrecognised"; +} + +function record(level, holder, seenThisEntry) { + const fields = (holder || {}).customFields || {}; + Object.keys(fields).forEach(function (name) { + const value = fields[name]; + if (isEmpty(value)) return; + if (!found[name]) { + found[name] = { level: level, entries: 0, values: 0, shapes: [], writingSystems: [], wsEntries: {} }; + } + const record_ = found[name]; + record_.values += 1; + const shape = shapeOf(value); + if (record_.shapes.indexOf(shape) === -1) record_.shapes.push(shape); + if (!seenThisEntry[name]) seenThisEntry[name] = {}; + if (shape === "multitext" || shape === "date" || shape === "number") { + Object.keys(value).forEach(function (ws) { + // An empty alternative is not use of that writing system, matching isEmpty() above. + if (isEmpty(value[ws])) return; + if (record_.writingSystems.indexOf(ws) === -1) record_.writingSystems.push(ws); + seenThisEntry[name][ws] = true; + }); + } + }); +} + +if (target.getCollectionNames().indexOf("lexicon") !== -1) { + target.lexicon.find({}, { customFields: 1, senses: 1 }).forEach(function (doc) { + const seenThisEntry = {}; + record("entry", doc, seenThisEntry); + (doc.senses || []).forEach(function (sense) { + if (!sense || typeof sense !== "object") return; + record("senses", sense, seenThisEntry); + (sense.examples || []).forEach(function (example) { + if (example && typeof example === "object") record("examples", example, seenThisEntry); + }); + }); + // entries counts entries, not values: a field used twice in one entry still counts once, and + // the same goes for each of its writing systems. + Object.keys(seenThisEntry).forEach(function (name) { + found[name].entries += 1; + Object.keys(seenThisEntry[name]).forEach(function (ws) { + found[name].wsEntries[ws] = (found[name].wsEntries[ws] || 0) + 1; + }); + }); + }); +} + +print(EJSON.stringify(found)); +""" + return mongosh_json(config, js, what=f"Scanning custom field usage in {db_name}") + + +def strip_custom_field_prefix(name: str, level: str) -> str: + """The bare field name, with LF's "customField__" prefix removed. + + LfMerge builds these names in ConvertUtilities.NormalizedFieldName as + customField__, so the level has to + come off as well as the "customField_" part. Whatever is left is the LCM field name with its + spaces still encoded; report_custom_fields() puts those back. + """ + for candidate in (f"{CUSTOM_FIELD_PREFIX}{level}_", CUSTOM_FIELD_PREFIX): + if name.startswith(candidate): + return name[len(candidate):] + return name + + +def describe_custom_fields(declared: dict[str, dict], used: dict[str, dict]) -> list[dict]: + """Merge the config and the data into one row per custom field, sorted for reporting. + + A field declared in the config takes its type from there. A field only found in the data has its + type guessed from the value shape, and is marked as a guess -- a guess is a reasonable basis for + a report and a poor basis for creating a field in FieldWorks, where the type is hard to undo. + """ + rows = [] + for name in sorted(set(declared) | set(used)): + config_entry = declared.get(name, {}) + usage = used.get(name, {}) + level = config_entry.get("level") or usage.get("level") or "?" + shapes = list(usage.get("shapes") or []) + + lf_type = config_entry.get("type") + guessed = False + if lf_type: + lcm_type = LCM_TYPE_FOR_LF_TYPE.get(lf_type) + elif len(shapes) == 1: + lf_type, lcm_type = LF_TYPE_FOR_VALUE_SHAPE.get(shapes[0], (None, None)) + guessed = True + else: + lf_type, lcm_type = None, None + guessed = bool(shapes) + + rows.append({ + "name": name, + "short_name": strip_custom_field_prefix(name, level), + "level": level, + "lf_type": lf_type, + "lcm_type": lcm_type, + "lcm_class": LCM_CLASS_FOR_LEVEL.get(level), + "guessed": guessed, + "declared": name in declared, + "label": config_entry.get("label"), + "list_code": config_entry.get("listCode"), + "input_systems": config_entry.get("inputSystems") or usage.get("writingSystems") or [], + "ws_entries": usage.get("wsEntries") or {}, + "entries": usage.get("entries", 0), + "values": usage.get("values", 0), + "shapes": shapes, + }) + order = {"entry": 0, "senses": 1, "examples": 2} + rows.sort(key=lambda row: (order.get(row["level"], 9), -row["entries"], row["name"])) + return rows + + +def report_custom_fields(rows: list[dict]) -> None: + """Print the custom fields, and what a future LexBox API would need in order to create them.""" + if not rows: + log("Custom fields: none") + return + + log(f"Custom fields ({len(rows)}):") + for row in rows: + detail = f"list={row['list_code']}" if row["list_code"] else ",".join(row["input_systems"]) + flags = [] + if row["guessed"]: + flags.append("GUESSED from data, not in config") + elif not row["declared"]: + flags.append("not in config") + if row["entries"] == 0: + flags.append("declared but unused") + if row["list_code"] and row["list_code"] not in STANDARD_LF_OPTION_LIST_CODES: + flags.append(f"option list '{row['list_code']}' is not one FieldWorks ships") + if row["lf_type"] is None: + flags.append(f"type undetermined (shapes seen: {', '.join(row['shapes']) or 'none'})") + log(f" {row['level']:8s} {row['short_name']:34s} {str(row['lf_type']):24s} " + f"{detail:18s} {row['entries']:6d} entries" + + (" [" + "; ".join(flags) + "]" if flags else "")) + + guessed = [row for row in rows if row["guessed"]] + lists_needed = sorted({row["list_code"] for row in rows + if row["list_code"] and row["list_code"] not in STANDARD_LF_OPTION_LIST_CODES}) + log("") + log(" To create these in LexBox, an API would need per field: LCM class, field name, type,") + log(" writing systems, and for list fields the possibility list. As derived here:") + for row in rows: + if row["lcm_type"] is None: + continue + field_name = row["short_name"].replace("_", " ") + log(f" {row['lcm_class'] or '?':19s} {field_name:34s} {row['lcm_type']:19s}" + + (f" list={row['list_code']}" if row["list_code"] else "") + + (" (type guessed)" if row["guessed"] else "")) + if guessed: + log("") + log(f" WARNING: {len(guessed)} field(s) above are typed by GUESS from the value shape, because") + log(" the project config does not declare them. Confirm each before anything creates") + log(" it in FieldWorks: a wrong custom field type is painful to undo once synced.") + if lists_needed: + log("") + log(f" WARNING: {len(lists_needed)} option list(s) are not ones FieldWorks ships and would have to be") + log(f" created first: {', '.join(lists_needed)}") + + +def plan_custom_field_renames(rows: list[dict]) -> tuple[list[dict], list[str]]: + """Work out which custom fields have to be renamed, and which case clashes are unhandled. + + Returns (renames, problems). Each rename carries the level, the Mongo key to rename and the key + to rename it to, plus the FieldWorks names either side for reporting. + + Two custom fields whose names differ only by case are a hazard, because FLEx is inconsistent + about whether it treats them as one field or two. Only the clashes named in + CASE_CLASH_CUSTOM_FIELD_RENAMES are handled; anything else is reported for a human rather than + guessed at, since which of the two should keep its name -- and whether they even hold the same + kind of data -- is not derivable from the names. + + Names are compared within a level. A field on LexEntry and one on LexSense do not collide even + when identically named, because FieldWorks keys a custom field by its class as well as its name. + """ + groups: dict[tuple[str, str], list[dict]] = {} + for row in rows: + groups.setdefault((row["level"], row["short_name"].lower()), []).append(row) + taken = set(groups) + + renames: list[dict] = [] + problems: list[str] = [] + for (level, _lowered), group in sorted(groups.items()): + spellings = {row["short_name"] for row in group} + if len(spellings) < 2: + continue + + renamed: list[dict] = [] + for row in sorted(group, key=lambda item: item["short_name"]): + new_name = CASE_CLASH_CUSTOM_FIELD_RENAMES.get(row["short_name"].replace("_", " ")) + if new_name is None: + continue + new_short = new_name.replace(" ", "_") + if (level, new_short.lower()) in taken: + problems.append( + f"{level}: cannot rename {row['short_name'].replace('_', ' ')!r} to " + f"{new_name!r} because the project already has a field of that name" + ) + continue + # Take the prefix off the row's own key rather than rebuilding it, so whatever spelling + # of customField__ this project uses is preserved exactly. + prefix = row["name"][: len(row["name"]) - len(row["short_name"])] + renames.append({ + "level": level, + "old_key": row["name"], + "new_key": prefix + new_short, + "old_name": row["short_name"].replace("_", " "), + "new_name": new_name, + }) + taken.add((level, new_short.lower())) + renamed.append(row) + + left = sorted(spellings - {row["short_name"] for row in renamed}) + if len(left) > 1: + problems.append( + f"{level}: these custom field names differ only by case and none of them is one " + f"this script knows how to rename: " + + ", ".join(repr(name.replace("_", " ")) for name in left) + ) + return renames, problems + + +def apply_custom_field_renames(rows: list[dict], renames: list[dict]) -> None: + """Rewrite the described rows in place so everything downstream sees the new names. + + The FieldWorks definitions -- and so FLExProject.CustomProperties -- are built from these rows. + A definition under the old name would not match the data the migration is about to write, and + LfMerge would skip the field. + """ + by_key = {(rename["level"], rename["old_key"]): rename for rename in renames} + for row in rows: + rename = by_key.get((row["level"], row["name"])) + if rename is not None: + row["name"] = rename["new_key"] + row["short_name"] = rename["new_name"].replace(" ", "_") + + +def report_custom_field_renames(renames: list[dict], problems: list[str]) -> None: + """Say what will be renamed, and what cannot be.""" + for rename in renames: + log(f" {rename['level']:8s} {rename['old_name']!r} -> {rename['new_name']!r}" + f" ({rename['old_key']} -> {rename['new_key']})") + for problem in problems: + log(f" ERROR: {problem}") + + +def rename_custom_fields_in_lexicon( + config: Config, db_name: str, renames: list[dict] +) -> dict: + """Rename custom field keys throughout db_name.lexicon, in the COPY only. + + Renames customFields and customFieldGuids alike, at whichever level the field lives at. The + whole document is rewritten rather than updated with $rename, which cannot reach a key inside + an array element; nothing else about it is touched, so dateModified and the dirtySR flags keep + saying when the entry was last really worked on. + + A container that somehow already holds the new key is reported and left alone rather than + overwritten, which would lose whichever value was there. + """ + if not renames: + return {"entriesChanged": 0, "keysRenamed": 0, "conflicts": []} + + js = f""" +const target = db.getSiblingDB({json.dumps(db_name)}); +const renames = {json.dumps(renames)}; +""" + """ +// The containers a rename applies to, by the level the field was declared at. +function containersFor(doc, level) { + if (level === "entry") return [doc]; + const senses = (doc.senses || []).filter(function (s) { return s && typeof s === "object"; }); + if (level === "senses") return senses; + const examples = []; + senses.forEach(function (sense) { + (sense.examples || []).forEach(function (example) { + if (example && typeof example === "object") examples.push(example); + }); + }); + return examples; +} + +let entriesChanged = 0; +let keysRenamed = 0; +const conflicts = []; + +target.lexicon.find({}).forEach(function (doc) { + let dirty = false; + renames.forEach(function (rename) { + containersFor(doc, rename.level).forEach(function (container) { + ["customFields", "customFieldGuids"].forEach(function (holder) { + const fields = container[holder]; + if (!fields || typeof fields !== "object") return; + if (!Object.prototype.hasOwnProperty.call(fields, rename.old_key)) return; + if (Object.prototype.hasOwnProperty.call(fields, rename.new_key)) { + conflicts.push({ id: doc._id.toString(), holder: holder, key: rename.new_key }); + return; + } + fields[rename.new_key] = fields[rename.old_key]; + delete fields[rename.old_key]; + keysRenamed += 1; + dirty = true; + }); + }); + }); + if (dirty) { + target.lexicon.replaceOne({ _id: doc._id }, doc); + entriesChanged += 1; + } +}); + +print(EJSON.stringify({ + entriesChanged: entriesChanged, + keysRenamed: keysRenamed, + conflicts: conflicts +})); +""" + result = mongosh_json(config, js, what=f"Renaming custom fields in {db_name}") + if result["conflicts"]: + raise Fatal( + f"{len(result['conflicts'])} container(s) already held the renamed custom field key, " + "so the rename was abandoned there rather than overwrite a value: " + + ", ".join(f"{c['id']}.{c['holder']}.{c['key']}" for c in result["conflicts"][:5]) + ) + log(f" {result['keysRenamed']} custom field key(s) renamed in " + f"{result['entriesChanged']} entry/entries") + return result + + +def derive_ws_selector( + field_ws: list[str], vernacular: list[str], analysis: list[str], ws_entries: dict[str, int] +) -> tuple[int, bool, str]: + """Pick FieldWorks' magic writing system selector for a text custom field. + + A field holding data in exactly one writing system gets the selector naming just that one, when + it leads a list: -2 for the first vernacular writing system, -1 for the first analysis one. Most + text custom fields in these projects are like this, and the single-writing-system selectors are + what makes them look right in FLEx rather than showing an empty row per writing system in a + list. A writing system leading both lists takes -2, since either selector resolves to it. + + Otherwise -4 covers every vernacular writing system of the project and -3 every analysis one, so + a field whose writing systems sit entirely inside one of the two lists gets that list's selector. + + A field that spans both gets a combined selector instead: -6 for every vernacular writing + system followed by every analysis one, -5 for the other order. Both cover the same set, so the + choice is only about which the user sees first, and that is settled by where the data actually + is -- the writing systems holding the most entries lead. FieldWorks still has no selector + meaning "these particular writing systems", so a combined selector brings in the project's + other writing systems too; that is harmless next to dropping the ones a single-sided selector + would leave out. + + The selector is not recoverable from LF: LfMerge expanded it into a plain list of writing + systems on the way in (ConvertLcmToMongoCustomField), so all that survives is where those + writing systems ended up. Returns (selector, confident, why not). + """ + vern, anal = set(vernacular), set(analysis) + tags = set(field_ws) + + # Checked before anything else, and against the DATA rather than the declaration: a field with + # one writing system's worth of data can be named exactly, and that beats any list selector. + with_data = [tag for tag, count in ws_entries.items() if count > 0] + if len(with_data) == 1: + only = with_data[0] + selector = None + if vernacular and only == vernacular[0]: + selector = WS_SELECTOR_FIRST_VERNACULAR + elif analysis and only == analysis[0]: + selector = WS_SELECTOR_FIRST_ANALYSIS + if selector is not None: + # Narrowing is possible: a field declaring several writing systems but holding data in + # one keeps only that one in FLEx. No data is lost, but say so, since it is a decision. + wider = sorted(tags - {only}) + if wider: + return (selector, False, + f"all its data is in '{only}', so {selector} names just that one; the " + f"config also lists {', '.join(wider)}, which the field will no longer offer") + return selector, True, "" + + if not tags: + return WS_SELECTOR_ALL_VERNACULAR, False, "the field declares no writing systems at all" + within_vernacular, within_analysis = tags <= vern, tags <= anal + if within_vernacular and not within_analysis: + return WS_SELECTOR_ALL_VERNACULAR, True, "" + if within_analysis and not within_vernacular: + return WS_SELECTOR_ALL_ANALYSIS, True, "" + if within_vernacular and within_analysis: + # Covered either way, so the selector is not in doubt; which side the field belongs to is, + # and nothing in the config can settle it. + return (WS_SELECTOR_ALL_VERNACULAR, False, + "every writing system it uses is both vernacular and analysis in this project") + + # Neither single-sided selector covers all of it. A combined one reaches both lists, so the + # only remaining question is which side leads: the one holding more of this field's data. A + # writing system on both sides counts for both and so cannot break the tie. + # Even a combined selector only reaches writing systems the project knows about, though. + uncovered = tags - vern - anal + if not (tags & vern) and not (tags & anal): + # Every writing system it uses is outside both lists, so no selector reaches any of them. + # The selector is not the thing to fix here; the project's writing systems are. + return (WS_SELECTOR_VERNACULAR_THEN_ANALYSIS, False, + f"none of its writing systems ({', '.join(sorted(tags))}) are in the project's " + f"vernacular or analysis lists, so no selector reaches them") + + vernacular_entries = sum(ws_entries.get(tag, 0) for tag in tags & vern) + analysis_entries = sum(ws_entries.get(tag, 0) for tag in tags & anal) + if analysis_entries > vernacular_entries: + selector, leading = WS_SELECTOR_ANALYSIS_THEN_VERNACULAR, "analysis" + else: + selector, leading = WS_SELECTOR_VERNACULAR_THEN_ANALYSIS, "vernacular" + + why = f"neither -3 nor -4 covers all of its writing systems, so {selector} takes in both lists" + if analysis_entries == vernacular_entries: + why += (f" and leads with the {leading} ones, the two sides holding the same amount of its " + f"data ({vernacular_entries} entries each)") + else: + why += (f" and leads with the {leading} ones, which hold more of its data " + f"({max(vernacular_entries, analysis_entries)} entries vs " + f"{min(vernacular_entries, analysis_entries)})") + if uncovered: + plural = len(uncovered) > 1 + why += (f"; {', '.join(sorted(uncovered))} " + f"{'are' if plural else 'is'} in neither list and " + f"{'stay' if plural else 'stays'} uncovered") + return selector, False, why + + +def plan_custom_field_definitions( + rows: list[dict], vernacular: list[str], analysis: list[str] +) -> tuple[list[dict], list[tuple[dict, str]]]: + """Turn described custom fields into FieldWorks definitions, and say what was left out. + + Returns (definitions, dropped). A definition carries the attributes FieldWorks writes and the + comment lines that belong above it; `dropped` pairs each omitted field with the reason, so the + caller can report rather than silently lose it. + """ + definitions: list[dict] = [] + dropped: list[tuple[dict, str]] = [] + + for row in rows: + lcm_type, lcm_class = row["lcm_type"], row["lcm_class"] + if lcm_type is None: + dropped.append((row, f"type undetermined (shapes seen: {', '.join(row['shapes']) or 'none'})")) + continue + if lcm_class is None: + dropped.append((row, f"unrecognised level {row['level']!r}")) + continue + + # LF never offered the single-string custom field type, so every text field it holds is a + # MultiUnicode that happens to have one writing system -- never a FieldWorks String. + is_text = lcm_type == "MultiUnicode" + if not is_text and row["entries"] == 0: + # A list or paragraph field with no data buys nothing and costs a possibility list that + # a new project may not even have. There is no data to lose by leaving it out. + dropped.append((row, "no data, and not a text field")) + continue + + name = row["short_name"].replace("_", " ") + attributes: dict[str, str] = { + "class": lcm_class, + "key": f"{lcm_class}{name}", + "name": name, + "type": lcm_type, + } + dest_class = LCM_DEST_CLASS_FOR_TYPE.get(lcm_type) + if dest_class is not None: + attributes["destclass"] = str(dest_class) + # LF's label is the only human-readable description that survives the round trip; FieldWorks + # keeps its own helpString, which LfMerge never carried across. Only worth writing when it + # says something the field name does not. + label = (row["label"] or "").strip() + if label and label != name: + attributes["helpString"] = label + + notes: list[str] = [] + needs_review = False + + if row["list_code"]: + list_root = LCM_LIST_ROOT_GUIDS.get(row["list_code"]) + if list_root is None: + list_root = LIST_ROOT_PLACEHOLDER + needs_review = True + notes.append( + f"REVIEW: option list {row['list_code']!r} has no known GUID in a newly created " + f"FieldWorks project. Create the list, then replace listRoot below with its GUID." + ) + attributes["listRoot"] = list_root + + if is_text: + selector, confident, why = derive_ws_selector( + row["input_systems"], vernacular, analysis, row["ws_entries"]) + attributes["wsSelector"] = str(selector) + if not confident: + needs_review = True + notes.append(f"REVIEW: wsSelector guessed as {selector}, because {why}.") + + if row["guessed"]: + needs_review = True + notes.append( + f"REVIEW: type GUESSED as {lcm_type} from the data (shapes seen: " + f"{', '.join(row['shapes']) or 'none'}); the LF config does not declare this field." + ) + + if needs_review: + notes.append(_ws_usage_note(row)) + + definitions.append({"row": row, "attributes": attributes, "notes": notes, + "needs_review": needs_review}) + + return definitions, dropped + + +def _ws_usage_note(row: dict) -> str: + """How many entries hold data in each writing system -- the evidence for checking a guess.""" + counts = row["ws_entries"] + if not counts: + return f"{row['entries']} entry/entries carry data, none of it in a named writing system" + detail = ", ".join( + f"{ws}={counts[ws]}" for ws in sorted(counts, key=lambda tag: (-counts[tag], tag)) + ) + return f"entries with data per writing system: {detail}" + + +def xml_comment_text(text: str) -> str: + """Make text safe inside an XML comment, which may not contain "--" nor end with "-".""" + text = " ".join(text.split()) + while "--" in text: + text = text.replace("--", "- -") + return f"{text} " if text.endswith("-") else text + + +def build_custom_properties_xml(definitions: list[dict]) -> str: + """Render FLExProject.CustomProperties exactly as FieldWorks writes it. + + Elements sorted by class then name, attributes alphabetical and one per line, tab indented, + CRLF throughout and no trailing newline -- all copied from real Chorus repositories so that the + first commit a human makes looks like one FieldWorks would have made. + """ + lines = [CUSTOM_PROPERTIES_HEADER, ""] + for definition in sorted( + definitions, key=lambda item: (item["attributes"]["class"], item["attributes"]["name"]) + ): + for note in definition["notes"]: + lines.append(f"\t") + lines.append("\t") + return CUSTOM_PROPERTIES_EOL.join(lines) + + +def write_custom_properties(xml: str, path: str) -> str: + """Write the XML to path, or path.1 / path.2 / ... Never overwrites an existing file. + + O_EXCL rather than an exists() check, so that two runs racing each other still get a file each. + newline="" keeps the CRLF endings already in the string from being translated again. + """ + candidate, index = path, 0 + while True: + try: + handle = os.open(candidate, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644) + except FileExistsError: + index += 1 + candidate = f"{path}.{index}" + continue + with os.fdopen(handle, "w", encoding="utf-8", newline="") as stream: + stream.write(xml) + return candidate + + +def report_custom_field_definitions( + definitions: list[dict], dropped: list[tuple[dict, str]] +) -> None: + """List the fields left out of the XML, and the ones whose definition is a guess.""" + if dropped: + log("") + log(f" {len(dropped)} field(s) deliberately NOT included:") + for row, reason in dropped: + log(f" {row['level']:8s} {row['short_name']:34s} {reason}") + review = [definition for definition in definitions if definition["needs_review"]] + if review: + log("") + log(f" {len(review)} definition(s) carry a REVIEW comment and need checking before you commit:") + for definition in review: + log(f" {definition['attributes']['class']:19s} {definition['attributes']['name']}") + + +def log_custom_properties_xml(xml: str) -> None: + """Print the XML between rules, so it can be read or copied straight out of the run.""" + log("") + log(f"--- {CUSTOM_PROPERTIES_FILENAME} " + "-" * 44) + # Printed with the platform's own line endings; a written file keeps the CRLF FieldWorks uses. + log(xml.replace(CUSTOM_PROPERTIES_EOL, "\n")) + log("-" * 78) + + +def emit_custom_properties( + definitions: list[dict], + dropped: list[tuple[dict, str]], + new_code: str, + lexbox_code: str, + lexbox_host: str, +) -> str: + """Write and print FLExProject.CustomProperties, and say what the human has to do with it.""" + xml = build_custom_properties_xml(definitions) + path = write_custom_properties(xml, f"{new_code}.{CUSTOM_PROPERTIES_FILENAME}") + + log("") + log("=" * 78) + log(f"STOPPING: {len(definitions)} custom field definition(s) must exist in FieldWorks first.") + log("=" * 78) + log("") + log("LfMerge does not create custom fields -- it skips them, and the data in them, with") + log("\"Custom field X from LF skipped, because we're not yet creating new custom fields in") + log("LCM\". The first Send/Receive is the only pass that would carry this project's data, so") + log("the definitions have to be in the repository before it runs. Nothing has been pushed.") + log("") + log(f"Written to: {os.path.abspath(path)}") + + report_custom_field_definitions(definitions, dropped) + + scheme, host = split_lexbox_host(lexbox_host) + log("") + log("To create them, commit the file to the project's Mercurial repository:") + log(f" hg clone {scheme}://{host}/hg/{lexbox_code} {lexbox_code}") + log(f" cp {path} {lexbox_code}/{CUSTOM_PROPERTIES_FILENAME}") + log(f" cd {lexbox_code} && hg commit -m 'Add custom fields' && hg push") + + log_custom_properties_xml(xml) + return path + + +def count_anchor_writing_system_usage(config: Config, db_name: str) -> dict[str, dict]: + """How many entries hold data in each writing system, split by vernacular or analysis field. + + Returns {tag: {"vernacular": n, "analysis": n}}, counting entries rather than values so a + writing system used three times in one entry still counts once, the same way the custom field + scan counts. + + The fields are the ones whose role is not in doubt: + + vernacular lexeme, citationForm, senses.examples.sentence + analysis senses.definition, senses.gloss, senses.examples.translation + + etymology is deliberately absent. In the 2026-07-06 corpus 741 projects configure it with + analysis writing systems and 378 with vernacular ones, so it is evidence of nothing. + """ + js = f""" +const target = db.getSiblingDB({json.dumps(db_name)}); +""" + """ +const counts = {}; + +function isFilled(alternative) { + return alternative !== null && typeof alternative === "object" + && alternative.value !== null && alternative.value !== undefined && alternative.value !== ""; +} + +// Records into a per-entry set, so an entry using a writing system repeatedly counts once. +function record(multitext, side, seen) { + if (!multitext || typeof multitext !== "object") return; + Object.keys(multitext).forEach(function (ws) { + if (isFilled(multitext[ws])) seen[side][ws] = true; + }); +} + +if (target.getCollectionNames().indexOf("lexicon") !== -1) { + target.lexicon.find({}).forEach(function (doc) { + const seen = { vernacular: {}, analysis: {} }; + record(doc.lexeme, "vernacular", seen); + record(doc.citationForm, "vernacular", seen); + (doc.senses || []).forEach(function (sense) { + if (!sense || typeof sense !== "object") return; + record(sense.definition, "analysis", seen); + record(sense.gloss, "analysis", seen); + (sense.examples || []).forEach(function (example) { + if (!example || typeof example !== "object") return; + record(example.sentence, "vernacular", seen); + record(example.translation, "analysis", seen); + }); + }); + ["vernacular", "analysis"].forEach(function (side) { + Object.keys(seen[side]).forEach(function (ws) { + if (!counts[ws]) counts[ws] = { vernacular: 0, analysis: 0 }; + counts[ws][side] += 1; + }); + }); + }); +} + +print(EJSON.stringify(counts)); +""" + return mongosh_json(config, js, what=f"Counting writing system usage in {db_name}") + + +def classify_writing_systems( + found: dict[str, list[str]], + override_vernacular: list[str] | None = None, + override_analysis: list[str] | None = None, + anchor_counts: dict[str, dict] | None = None, +) -> tuple[list[str], list[str], list[str]]: + """Split the project's writing systems into vernacular and analysis. + + Returns (vernacular, analysis, unclassified). + + Every writing system the project knows about has to reach the FieldWorks + project, or data in it could not sync. Anything that is neither vernacular + nor analysis by the field conventions is reported as `unclassified` and + appended to the analysis list, which is the safer of the two: an extra + analysis writing system is inert, whereas a missing one loses data. + + The overrides replace the derived base lists, but coverage still applies -- + uncovered writing systems are appended to analysis either way, since the + whole point of the check is that the union is complete. + + A writing system in both lists is then removed from one of them, so that each writing system + has a single role in the new FieldWorks project. LF has no such constraint, so the overlap is + common: one project lists "rw" and "en" in its lexeme field and both again in its + definition. A writing system in both lists also tells nothing about the role of any field that + uses it, which is what leaves custom field wsSelectors undecidable. + + Which list it leaves is decided by `anchor_counts` -- where its data actually is, per + count_anchor_writing_system_usage -- and not by which field the config declares it on, since a + declaration can rest on a single stray value. Without counts, or on a tie, vernacular keeps it. + The vernacular list is never emptied, since LexBox requires one, and an explicitly overridden + list is never trimmed: --ws-vernacular and --ws-analysis say what the lists are to be. + VERNACULAR_ANALYSIS_OVERLAP_EXEMPTION is never removed from analysis. + + If the analysis list leads with an unlisted-language (qaa) writing system it + is reordered to lead with ANTHRO_DEFAULT_ANALYSIS_WS instead, and says so in + the log. The first entry becomes the new FieldWorks project's default + analysis writing system, and liblcm cannot write the anthropology list in a + qaa one. `en` is added if the project does not have it; an extra analysis + writing system is inert, and without it the project cannot be created. Any + other leading writing system is left where the project put it. + """ + if override_vernacular: + vernacular = dedup(override_vernacular) + else: + vernacular = dedup(found["lexeme"] + found["citationForm"]) + + if override_analysis: + analysis = dedup(override_analysis) + else: + analysis = dedup(found["definition"] + found["gloss"]) + + every = dedup(found["config_all"] + found["declared"]) + classified = set(vernacular) | set(analysis) + unclassified = [tag for tag in every if tag not in classified] + if unclassified: + analysis = dedup(analysis + unclassified) + + # Before the qaa check below, so that runs against the list the project will actually get. + counts = anchor_counts or {} + for tag in [tag for tag in analysis if tag in set(vernacular)]: + usage = counts.get(tag) or {} + in_vernacular = usage.get("vernacular", 0) + in_analysis = usage.get("analysis", 0) + evidence = f"{in_vernacular} vernacular vs {in_analysis} analysis entries" + + # Whichever side holds less of the writing system's data is the side it leaves. A tie -- + # which includes a writing system with no data at all -- leaves the vernacular list alone. + leaves = "vernacular" if in_analysis > in_vernacular else "analysis" + keep_reason = None + forced_reason = None + + if override_vernacular and override_analysis: + keep_reason = "both lists were given explicitly" + elif leaves == "vernacular" and override_vernacular: + leaves = "analysis" + forced_reason = "--ws-vernacular says what the vernacular list is to be" + elif leaves == "analysis" and override_analysis: + leaves = "vernacular" + forced_reason = "--ws-analysis says what the analysis list is to be" + + # LexBox requires a vernacular writing system, so the last one never leaves that list. + if keep_reason is None and leaves == "vernacular" and len(vernacular) == 1: + leaves = "analysis" + forced_reason = "it is the project's only vernacular writing system, which LexBox requires" + + # The anthropology list can only be written in this one, so it never leaves analysis. + if keep_reason is None and leaves == "analysis" and tag == VERNACULAR_ANALYSIS_OVERLAP_EXEMPTION: + keep_reason = "the anthropology list needs it and can be written in no other" + + if keep_reason is not None: + log(f" NOTE: '{tag}' is in both writing system lists ({evidence}); left in both,") + log(f" because {keep_reason}.") + continue + + if leaves == "vernacular": + vernacular = [item for item in vernacular if item != tag] + else: + analysis = [item for item in analysis if item != tag] + + # Compare against the side it actually left, not the side the data preferred: a guard above + # may have overruled the data, and saying "where less of its data is" would then be false. + left_behind = in_vernacular if leaves == "vernacular" else in_analysis + kept = in_analysis if leaves == "vernacular" else in_vernacular + log(f" NOTE: '{tag}' was in both writing system lists ({evidence}); removed from {leaves},") + if left_behind > kept: + because = f", because {forced_reason}" if forced_reason else "" + log(f" even though that is where more of its data is{because}.") + else: + log(" which is where less of its data is.") + + if not analysis: + log(" WARNING: no analysis writing systems are left. LexBox will fall back to") + log(f" '{ANTHRO_DEFAULT_ANALYSIS_WS}'; pass --ws-analysis to choose instead.") + + # Last, because the first analysis writing system is the one liblcm writes the anthropology + # list in, and it cannot write it in a qaa one. Applied even to an explicit --ws-analysis: the + # constraint is liblcm's, not a preference, and getting it wrong fails the whole creation. + if analysis and analysis[0].lower().startswith(UNLISTED_LANGUAGE_PREFIX): + leading = analysis[0] + analysis = [ANTHRO_DEFAULT_ANALYSIS_WS] + [ + tag for tag in analysis if tag != ANTHRO_DEFAULT_ANALYSIS_WS + ] + log(f" NOTE: analysis writing systems led with '{leading}'; moved " + f"'{ANTHRO_DEFAULT_ANALYSIS_WS}' to the front so that the new project has a default") + log(" analysis writing system the anthropology list can be written in.") + + return vernacular, analysis, unclassified + + +def count_collections(config: Config, db_name: str) -> int: + js = f""" +print(JSON.stringify({{ count: db.getSiblingDB({json.dumps(db_name)}).getCollectionNames().length }})); +""" + return int(mongosh_json(config, js, what=f"Inspecting database {db_name}")["count"]) + + +def copy_project_database(config: Config, source_db: str, target_db: str, *, drop: bool) -> None: + """Copy every collection and index of source_db into target_db. + + Uses mongodump piped through mongorestore --nsFrom/--nsTo so that BSON + types, indexes and collection options are preserved exactly -- no + document-by-document marshalling as backup.mjs has to do. + """ + require_tool("mongodump") + require_tool("mongorestore") + + with tempfile.TemporaryDirectory(prefix="lf-migrate-") as tmpdir: + archive = os.path.join(tmpdir, f"{source_db}.archive") + + log(f" dumping {source_db}...") + run( + ["mongodump", f"--uri={config.uri(source_db)}", f"--archive={archive}", "--quiet"], + what=f"mongodump of {source_db}", + ) + + if not os.path.exists(archive) or os.path.getsize(archive) == 0: + log(f" {source_db} is empty; nothing to restore") + return + + log(f" restoring into {target_db}...") + cmd = [ + "mongorestore", + f"--uri={config.uri()}", + f"--archive={archive}", + f"--nsFrom={source_db}.*", + f"--nsTo={target_db}.*", + "--quiet", + ] + if drop: + cmd.append("--drop") + run(cmd, what=f"mongorestore into {target_db}") + + +# A numeric BSON value does not always reach mongosh as a JS number: int32 and +# double do, but int64 and decimal arrive as wrapper objects. Shared by both +# coercion scripts below, which is why it is a separate string and not inlined. +NUMERIC_HELPERS_JS = """ +function isNumeric(value) { + if (typeof value === "number") return true; + if (value === null || typeof value !== "object") return false; + const kind = value._bsontype; + return kind === "Int32" || kind === "Long" || kind === "Double" || kind === "Decimal128"; +} + +function asString(value) { + return typeof value === "number" ? String(value) : value.toString(); +} +""" + + +def coerce_option_list_keys(config: Config, db_name: str) -> dict: + """Rewrite every numeric option list item key in db_name as a string. + + Runs against the COPY, straight after the database duplication, so the + original project keeps its integers. `items[].key` is the only field in an + optionlists document that is ever numeric in the corpus; `defaultItemKey` + names an item key too, so it is coerced alongside them, defensively -- no + project in the 2026-07-06 dump has a numeric one. + + A coerced key that duplicates a key already in the same list would make that + list ambiguous. No such case exists in the corpus, so this reports any it + finds rather than refusing to migrate, and still performs the coercion. + """ + js = f""" +const targetName = {json.dumps(db_name)}; +""" + NUMERIC_HELPERS_JS + """ +const target = db.getSiblingDB(targetName); + +let lists = 0; +let listsChanged = 0; +let keysCoerced = 0; +let defaultKeysCoerced = 0; +const collisions = []; +const samples = []; + +if (target.getCollectionNames().indexOf("optionlists") !== -1) { + target.optionlists.find({}).forEach(function (doc) { + lists += 1; + + const items = Array.isArray(doc.items) ? doc.items : []; + // Keys already spoken for, so a coerced key landing on one of them can be + // spotted instead of silently creating a duplicate. + const taken = new Set(); + items.forEach(function (item) { + if (item && typeof item.key === "string") taken.add(item.key); + }); + + let coercedHere = 0; + const newItems = items.map(function (item) { + if (!item || !isNumeric(item.key)) return item; + const key = asString(item.key); + if (taken.has(key)) { + collisions.push({ code: doc.code, key: key, value: item.value }); + } + taken.add(key); + coercedHere += 1; + if (samples.length < 5) { + samples.push({ code: doc.code, key: key, value: item.value }); + } + // Object.assign keeps the field order, and `key` already exists on the + // item, so it is replaced in place rather than moved to the end. + return Object.assign({}, item, { key: key }); + }); + + const update = {}; + if (coercedHere > 0) { + update.items = newItems; + keysCoerced += coercedHere; + } + if (isNumeric(doc.defaultItemKey)) { + update.defaultItemKey = asString(doc.defaultItemKey); + defaultKeysCoerced += 1; + } + if (Object.keys(update).length > 0) { + target.optionlists.updateOne({ _id: doc._id }, { $set: update }); + listsChanged += 1; + } + }); +} + +print(EJSON.stringify({ + lists: lists, + listsChanged: listsChanged, + keysCoerced: keysCoerced, + defaultKeysCoerced: defaultKeysCoerced, + collisions: collisions, + samples: samples, +})); +""" + result = mongosh_json(config, js, what=f"Coercing option list keys in {db_name}") + + log(f" {result['lists']} option list(s) examined") + if result["keysCoerced"]: + log(f" {result['keysCoerced']} numeric key(s) coerced to strings " + f"across {result['listsChanged']} list(s)") + for sample in result["samples"]: + log(f" {sample['code']}: {sample['key']!r} ({sample['value']})") + else: + log(" no numeric keys found; nothing to coerce") + if result["defaultKeysCoerced"]: + log(f" {result['defaultKeysCoerced']} numeric defaultItemKey(s) coerced") + if result["collisions"]: + log("") + log("WARNING: coercing produced a key that the list already had as a string.") + log(" The list now has two items with the same key, and a reference to") + log(" it is ambiguous. Inspect these before pushing to LexBox:") + for collision in result["collisions"]: + log(f" {collision['code']}: key {collision['key']!r}") + log("") + return result + + +def coerce_special_lexicon_references(config: Config, db_name: str, source_code: str) -> dict: + """Coerce the two known numeric lexicon references, and report any others. + + Only the sense named in SPECIAL_LEXICON_INT_REFERENCES for this project code + is rewritten, and only when its `values` list is exactly the expected one. + Every other numeric reference -- in another project, another field, or a + values list that does not match -- is counted and reported, never touched. + + Nothing here matters unless coerce_option_list_keys has already run: it is + the list-side coercion that leaves these two references dangling. + """ + special = SPECIAL_LEXICON_INT_REFERENCES.get(source_code) + field, expected = special if special else ("", []) + + js = f""" +const targetName = {json.dumps(db_name)}; +const specialField = {json.dumps(field)}; +const specialValues = {json.dumps(expected)}; +""" + NUMERIC_HELPERS_JS + """ +const target = db.getSiblingDB(targetName); + +// Exact match against the expected list: same length, every entry numeric, and +// every entry the same number. That is what makes this a special case for two +// known senses rather than a blanket lexicon rewrite. +function isSpecialList(values) { + if (!specialField || specialValues.length === 0) return false; + if (!Array.isArray(values) || values.length !== specialValues.length) return false; + for (let index = 0; index < values.length; index += 1) { + if (!isNumeric(values[index])) return false; + if (asString(values[index]) !== String(specialValues[index])) return false; + } + return true; +} + +let entriesChanged = 0; +let valuesCoerced = 0; +const coerced = []; +const untouched = []; + +if (target.getCollectionNames().indexOf("lexicon") !== -1) { + // Only `senses` is read or written, so leave the rest of each entry on the + // server -- a lexicon can run to tens of thousands of documents. + target.lexicon.find({}, { senses: 1 }).forEach(function (doc) { + const senses = Array.isArray(doc.senses) ? doc.senses : []; + let changed = false; + + senses.forEach(function (sense, senseIndex) { + if (!sense || typeof sense !== "object") return; + Object.keys(sense).forEach(function (fieldName) { + const holder = sense[fieldName]; + if (!holder || typeof holder !== "object" || !Array.isArray(holder.values)) return; + if (!holder.values.some(isNumeric)) return; + + const shown = holder.values.map(function (value) { + return isNumeric(value) ? asString(value) : value; + }); + + if (fieldName !== specialField || !isSpecialList(holder.values)) { + untouched.push({ + entry: doc._id.toString(), + sense: senseIndex, + field: fieldName, + values: shown, + }); + return; + } + + holder.values = holder.values.map(function (value) { + return isNumeric(value) ? asString(value) : value; + }); + valuesCoerced += holder.values.length; + coerced.push({ + entry: doc._id.toString(), + sense: senseIndex, + field: fieldName, + values: shown, + }); + changed = true; + }); + }); + + if (changed) { + target.lexicon.updateOne({ _id: doc._id }, { $set: { senses: senses } }); + entriesChanged += 1; + } + }); +} + +print(EJSON.stringify({ + entriesChanged: entriesChanged, + valuesCoerced: valuesCoerced, + coerced: coerced, + untouched: untouched, +})); +""" + result = mongosh_json(config, js, what=f"Coercing lexicon references in {db_name}") + + if special: + log(f" {source_code!r} is a known special case: " + f"{field}.values == {expected}") + if result["coerced"]: + for record in result["coerced"]: + log(f" coerced entry {record['entry']} sense #{record['sense']} " + f"{record['field']}.values {record['values']} -> strings") + log(f" {result['valuesCoerced']} value(s) in {result['entriesChanged']} entry/entries") + elif special: + log(" expected special-case sense was NOT found; nothing coerced") + log(" (the project data may have changed since the dump this was derived from)") + else: + log(" not a special-case project; no lexicon value was changed") + + if result["untouched"]: + log("") + log(f"WARNING: {len(result['untouched'])} numeric option list reference(s) in the") + log(" lexicon were left as they are, because they are not one of the two") + log(" known special cases. They will not resolve against the coerced") + log(" option lists. Add them to SPECIAL_LEXICON_INT_REFERENCES if they") + log(" should be converted too:") + for record in result["untouched"][:10]: + log(f" entry {record['entry']} sense #{record['sense']} " + f"{record['field']}.values = {record['values']}") + if len(result["untouched"]) > 10: + log(f" ... and {len(result['untouched']) - 10} more") + log("") + return result + + + +def find_empty_entry_guids(config: Config, db_name: str) -> list[dict]: + """Return the lexicon entries in db_name whose guid is an empty string. + + Matches `guid: ""` exactly, because that is the only shape the problem takes + in the 2026-07-06 corpus: five projects with one such entry each, no nulls, + no missing guid fields and no malformed values. An entry whose guid is + merely unusual -- a UUID that is not version 4, say -- is left alone, since + LfMerge parses it happily. + + The lexeme form comes back with each hit purely so the log can name the + entry instead of only its ObjectId. Read-only, so it is safe to point at the + source database to preview what a run would do. + """ + js = f""" +const target = db.getSiblingDB({json.dumps(db_name)}); +const found = []; +if (target.getCollectionNames().indexOf("lexicon") !== -1) {{ + target.lexicon.find({{ guid: "" }}, {{ lexeme: 1 }}).forEach(function (doc) {{ + // lexeme is a multitext, {{ "": {{ "value": "..." }} }}. Any one form is + // enough to identify the entry in a log line. + let label = ""; + const lexeme = doc.lexeme || {{}}; + Object.keys(lexeme).forEach(function (ws) {{ + if (!label && lexeme[ws] && lexeme[ws].value) {{ + label = ws + ": " + lexeme[ws].value; + }} + }}); + found.push({{ id: doc._id.toString(), label: label }}); + }}); +}} +print(EJSON.stringify({{ found: found }})); +""" + return list(mongosh_json(config, js, what=f"Finding empty entry guids in {db_name}")["found"]) + + +def fix_empty_entry_guids(config: Config, db_name: str) -> dict: + """Give every lexicon entry with an empty-string guid a fresh UUID4. + + Runs against the COPY, so the original project keeps its empty guid. + + An empty guid does not make LfMerge fail, which is why it goes unnoticed. + LfMerge reads the field with Guid.TryParse (MongoConnection.CanParseGuid), + so such an entry is quietly dropped from the ObjectId->Guid map that + comments are attached by, and ConvertMongoToLcmLexicon then falls back to + `lfEntry.Guid ?? Guid.Empty`. The entry would be created in FieldWorks under + the all-zeros GUID, and any later entry in the same state would find that + object and overwrite it instead of creating its own. Fixing this before the + first Send/Receive is what keeps the entries distinct. + + The new guids are generated here rather than in mongosh: uuid.uuid4() + produces exactly what Guid::create() does in PHP -- lowercase, hyphenated, + version 4 -- which is the only form Guid::isValid accepts. mongosh's UUID() + would produce BSON binary, the wrong type for this field. + + Only `guid` is written. dateModified and dirtySR are left alone: restoring an + identity is not a content edit, and touching them would misreport when the + entry was last worked on. + """ + entries = find_empty_entry_guids(config, db_name) + if not entries: + log(" no entries with an empty guid") + return {"fixed": 0, "assignments": []} + + # The unique {guid: 1} index (LexEntryModel.php:515) would reject a duplicate + # part way through the batch, so make sure this batch is distinct with itself. + seen: set[str] = set() + assignments = [] + for entry in entries: + guid = str(uuid.uuid4()) + while guid in seen: + guid = str(uuid.uuid4()) + seen.add(guid) + assignments.append({"id": entry["id"], "guid": guid, "label": entry.get("label", "")}) + + js = f""" +const target = db.getSiblingDB({json.dumps(db_name)}); +const assignments = {json.dumps(assignments)}; +const updated = []; +const collisions = []; + +assignments.forEach(function (assignment) {{ + // Cheap insurance: a collision would be unrecoverable half way through a + // batch, and this costs one indexed lookup per entry. + if (target.lexicon.findOne({{ guid: assignment.guid }}) !== null) {{ + collisions.push(assignment.guid); + return; + }} + // The guid: "" in the filter means a document that somehow already got a + // guid is left alone rather than overwritten. + const result = target.lexicon.updateOne( + {{ _id: new ObjectId(assignment.id), guid: "" }}, + {{ $set: {{ guid: assignment.guid }} }} + ); + if (result.modifiedCount === 1) {{ + updated.push(assignment.id); + }} +}}); + +print(EJSON.stringify({{ updated: updated, collisions: collisions }})); +""" + result = mongosh_json(config, js, what=f"Assigning entry guids in {db_name}") + + if result["collisions"]: + raise Fatal( + "generated a guid that the lexicon already contains, which should not be possible: " + + ", ".join(result["collisions"]) + ) + if len(result["updated"]) != len(assignments): + raise Fatal( + f"expected to give guids to {len(assignments)} entry/entries but updated " + f"{len(result['updated'])}; the copy may have changed underneath us" + ) + + for assignment in assignments: + label = f" ({assignment['label']})" if assignment["label"] else "" + log(f" entry {assignment['id']}{label} -> {assignment['guid']}") + log(f" {len(assignments)} empty guid(s) replaced") + return {"fixed": len(assignments), "assignments": assignments} + + +def create_project_record( + config: Config, + source_code: str, + new_code: str, + new_name: str, + identifier: str, + renames: list[dict] | None = None, +) -> str: + """Insert the copied project document and register it with its members. + + Returns the new project's _id as a hex string. Called AFTER the project + database has been copied, so there is never a window in which the project + exists but its lexicon database does not (same ordering as backup.mjs). + + `renames` applies custom field renames to the copied config. It has to happen here rather than + in a step of its own because the document is copied wholesale from the source, config included, + so this is the only moment at which the new project's config exists but nothing has read it. + A custom field name appears in the config both as a key -- under entry.fields, senses.fields and + each roleViews..fields -- and as a value, in the fieldOrder lists, so both are rewritten. + """ + js = f""" +const main = db.getSiblingDB({json.dumps(config.database)}); +const renames = {json.dumps(renames or [])}; +const doc = main.projects.findOne({{ projectCode: {json.dumps(source_code)} }}); +if (doc === null) {{ + throw new Error("source project disappeared: " + {json.dumps(source_code)}); +}} + +delete doc._id; + +// Rename custom fields wherever the config mentions them, as a key or as a fieldOrder entry. +// Object.keys() is a snapshot, so re-keying while walking it is safe. +function renameThroughout(node, oldKey, newKey) {{ + if (Array.isArray(node)) {{ + for (let i = 0; i < node.length; i++) {{ + if (node[i] === oldKey) {{ node[i] = newKey; }} + else {{ renameThroughout(node[i], oldKey, newKey); }} + }} + }} else if (node !== null && typeof node === "object") {{ + Object.keys(node).forEach(function (key) {{ + if (key === oldKey) {{ + node[newKey] = node[key]; + delete node[key]; + renameThroughout(node[newKey], oldKey, newKey); + }} else {{ + renameThroughout(node[key], oldKey, newKey); + }} + }}); + }} +}} +// A field whose label still reads the old name is only half renamed, and the label is what LF +// shows and what this script would emit as the FieldWorks helpString. Only a label that matches +// the old name is rewritten, so a label somebody deliberately set to something else survives. +function relabel(node, key, oldLabel, newLabel) {{ + if (Array.isArray(node)) {{ + node.forEach(function (item) {{ relabel(item, key, oldLabel, newLabel); }}); + }} else if (node !== null && typeof node === "object") {{ + Object.keys(node).forEach(function (k) {{ + const child = node[k]; + if (k === key && child !== null && typeof child === "object" && child.label === oldLabel) {{ + child.label = newLabel; + }} + relabel(child, key, oldLabel, newLabel); + }}); + }} +}} +let configKeysRenamed = 0; +renames.forEach(function (rename) {{ + const before = JSON.stringify(doc.config || {{}}); + renameThroughout(doc.config, rename.old_key, rename.new_key); + relabel(doc.config, rename.new_key, rename.old_name, rename.new_name); + if (JSON.stringify(doc.config || {{}}) !== before) {{ configKeysRenamed += 1; }} +}}); +doc.projectCode = {json.dumps(new_code)}; +doc.projectName = {json.dumps(new_name)}; +// Point the copy at its own (new) Send/Receive project. The identifier is +// derived from the projectCode but normalised for LexBox, so the two differ +// whenever the projectCode has capitals or underscores. +doc.sendReceiveProjectIdentifier = {json.dumps(identifier)}; + +const inserted = main.projects.insertOne(doc); +const newId = inserted.insertedId; + +// Give every member of the original project access to the copy as well. +const memberIds = Object.keys(doc.users || {{}}).map((id) => new ObjectId(id)); +let membersUpdated = 0; +if (memberIds.length) {{ + membersUpdated = main.users.updateMany( + {{ _id: {{ $in: memberIds }} }}, + {{ $addToSet: {{ projects: newId }} }} + ).modifiedCount; +}} + +print(EJSON.stringify({{ + id: newId.toString(), + membersUpdated: membersUpdated, + configKeysRenamed: configKeysRenamed, +}})); +""" + result = mongosh_json(config, js, what="Creating project record") + log(f" new project _id: {result['id']}") + log(f" granted access to {result['membersUpdated']} member(s)") + if renames: + renamed = result.get("configKeysRenamed", 0) + log(f" renamed {renamed} custom field(s) in the copied config") + if renamed != len(renames): + raise Fatal( + f"expected to rename {len(renames)} custom field(s) in the config but changed " + f"{renamed}; the project config is not shaped as this script expects" + ) + return result["id"] + + +def delete_project_record(config: Config, project_id: str) -> None: + mongosh_json( + config, + f""" +const main = db.getSiblingDB({json.dumps(config.database)}); +const doomed = new ObjectId({json.dumps(project_id)}); +main.users.updateMany({{ projects: doomed }}, {{ $pull: {{ projects: doomed }} }}); +main.projects.deleteOne({{ _id: doomed }}); +print(JSON.stringify({{ ok: 1 }})); +""", + what="Removing project record", + ) + + +# --------------------------------------------------------------------------- # +# asset steps +# --------------------------------------------------------------------------- # + + +def stage_assets(config: Config, app_name: str, source_db: str, target_db: str) -> str | None: + """Copy audio/pictures to the new project's own assets folder, as real dirs. + + A non-Send/Receive project keeps these as ordinary directories + (LexProjectModel::createAssetsFolders takes its `else` branch), so this is a + plain recursive copy. Symlinks are dereferenced anyway, defensively: they + may exist and even be broken if the project was ever converted + (cf. `tar chf` in backup.mjs). + + These directories are turned into symlinks into the webwork tree later, by + link_assets_into_webwork(), once the first Send/Receive has created it. + """ + source_dir = config.assets_path(app_name, source_db) + target_dir = config.assets_path(app_name, target_db) + + if not os.path.isdir(source_dir): + log(f" no assets directory at {source_dir}; skipping") + return None + + os.makedirs(target_dir, exist_ok=True) + total = 0 + for subfolder, _ in ASSET_LINKED_FILES: + source_sub = os.path.join(source_dir, subfolder) + target_sub = os.path.join(target_dir, subfolder) + # os.path.isdir follows symlinks and is False for a broken one. + if not os.path.isdir(source_sub): + log(f" {subfolder}: missing or broken symlink; creating empty directory") + os.makedirs(target_sub, exist_ok=True) + continue + if os.path.exists(target_sub) or os.path.islink(target_sub): + if os.path.islink(target_sub): + os.unlink(target_sub) + else: + shutil.rmtree(target_sub) + shutil.copytree(source_sub, target_sub, symlinks=False) + count = sum(len(files) for _, _, files in os.walk(target_sub)) + log(f" {subfolder}: {count} file(s) copied") + total += count + + chown_to_web_user(target_dir) + if total == 0: + log(" (source project had no asset files)") + return target_dir + + +def copy_missing_files(source_dir: str, target_dir: str) -> tuple[int, int]: + """Recursively copy files from source_dir into target_dir, skipping existing. + + "Any assets NOT found in the LinkedFiles folder get copied in" -- anything + the Send/Receive already brought down wins, so a re-run is a no-op. + """ + copied = skipped = 0 + for root, _dirnames, filenames in os.walk(source_dir): + relative = os.path.relpath(root, source_dir) + destination_root = target_dir if relative == "." else os.path.join(target_dir, relative) + os.makedirs(destination_root, exist_ok=True) + for name in filenames: + destination = os.path.join(destination_root, name) + if os.path.exists(destination): + skipped += 1 + else: + shutil.copy2(os.path.join(root, name), destination) + copied += 1 + return copied, skipped + + +def link_assets_into_webwork(config: Config, app_name: str, target_db: str, new_code: str) -> None: + """Move staged assets into the webwork LinkedFiles tree and symlink to them. + + This is the Python equivalent of + LexProjectModel::moveExistingFilesAndCreateSymlink, and leaves the copy with + the on-disk layout a real Send/Receive project has: + + assets//sf_/audio -> webwork//LinkedFiles/AudioVisual + assets//sf_/pictures -> webwork//LinkedFiles/Pictures + + Must run AFTER the first Send/Receive, which is what creates webwork/. + """ + webwork = config.webwork_path(new_code) + if not os.path.isdir(webwork): + raise Fatal( + f"Send/Receive work folder {webwork} does not exist. The first Send/Receive " + "does not appear to have cloned the project, so there is nowhere to put the " + "assets. Check `docker compose logs lfmerge`." + ) + + assets_dir = config.assets_path(app_name, target_db) + for subfolder, linked_subfolder in ASSET_LINKED_FILES: + asset_path = os.path.join(assets_dir, subfolder) + linked_path = os.path.join(config.linked_files_path(new_code), linked_subfolder) + + os.makedirs(linked_path, exist_ok=True) + + if os.path.islink(asset_path): + if os.path.realpath(asset_path) == os.path.realpath(linked_path): + log(f" {subfolder}: already symlinked into webwork") + continue + log(f" {subfolder}: replacing symlink to {os.path.realpath(asset_path)}") + os.unlink(asset_path) + elif os.path.isdir(asset_path): + copied, skipped = copy_missing_files(asset_path, linked_path) + log(f" {subfolder}: {copied} file(s) moved into webwork, {skipped} already there") + shutil.rmtree(asset_path) + elif os.path.exists(asset_path): + os.unlink(asset_path) + else: + log(f" {subfolder}: nothing staged") + + os.makedirs(os.path.dirname(asset_path), exist_ok=True) + os.symlink(linked_path, asset_path) + + # LFMerge commits these as www-data, so it must own them. + chown_to_web_user(config.linked_files_path(new_code)) + chown_to_web_user(assets_dir) + + +# --------------------------------------------------------------------------- # +# send/receive steps +# --------------------------------------------------------------------------- # + + +def split_lexbox_host(host: str) -> tuple[str, str]: + """Split a --lexbox-host value into (scheme, netloc). + + A bare host like "lexbox.org" gets https, since LexBox communication is over + HTTPS. An explicit scheme is honoured, which is how a development LexBox is + reached: it serves plain HTTP, with no certificate at all. + """ + raw = (host or "").strip().rstrip("/") + if not raw: + raise Fatal("--lexbox-host must not be empty") + if "//" in raw: + parts = urlsplit(raw) + scheme, netloc = (parts.scheme or "https"), parts.netloc + else: + scheme, netloc = "https", raw + if not netloc: + raise Fatal(f"could not parse a hostname out of --lexbox-host {host!r}") + if scheme not in ("http", "https"): + raise Fatal(f"unsupported scheme {scheme!r} in --lexbox-host {host!r}") + return scheme, netloc + + +class LexboxClient: + """Minimal LexBox API client, standard library only. + + Login is POST /api/login (LoginController.cs:21 sets the route to /api/login, + and the action has a bare [HttpPost]). The response BODY is the LexAuthUser + record; the JWT itself comes back only as the `.LexBoxAuth` cookie, because + the controller calls HttpContext.SignInAsync and AuthKernel.cs:229 replaces + the cookie ticket format with a JWT writer. + """ + + def __init__( + self, + host: str, + username: str, + password: str, + *, + auth_mode: str = "cookie", + timeout: int = DEFAULT_LEXBOX_TIMEOUT_SECONDS, + ) -> None: + self.scheme, self.netloc = split_lexbox_host(host) + self.base_url = f"{self.scheme}://{self.netloc}" + self.hostname = urlsplit(f"//{self.netloc}").hostname or self.netloc + self.username = username + self.password = password + self.auth_mode = auth_mode + self.timeout = timeout + self.cookie_jar = http.cookiejar.CookieJar() + self.jwt: str | None = None + + # -- transport --------------------------------------------------------- # + + @property + def is_localhost(self) -> bool: + """True for localhost and any name under the reserved .localhost TLD. + + The subdomains matter because a development LexBox is reached at + admin.localhost, not localhost. RFC 6761 reserves .localhost for local + use, so treating the whole family as local cannot loosen anything for a + real host such as lexbox.org. + """ + host = (self.hostname or "").lower() + return host == "localhost" or host.endswith(".localhost") + + def ssl_context(self) -> ssl.SSLContext: + """Verify certificates everywhere except the localhost family. + + A local dev LexBox may present a self-signed certificate, which is + accepted if and only if the host is localhost or a .localhost subdomain. + For any other host the full chain and hostname are verified, so a + self-signed or mismatched certificate is refused. + """ + context = ssl.create_default_context() + if self.is_localhost: + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + return context + + def _opener(self) -> urllib.request.OpenerDirector: + return urllib.request.build_opener( + urllib.request.HTTPSHandler(context=self.ssl_context()), + urllib.request.HTTPCookieProcessor(self.cookie_jar), + ) + + def _send( + self, + method: str, + path: str, + *, + query: list[tuple[str, str]] | None = None, + json_body: Any = None, + authenticate: bool = False, + ) -> tuple[int, Any, bytes]: + params = list(query or []) + headers: dict[str, str] = {} + + if authenticate: + if not self.jwt: + raise Fatal("internal error: authenticated request attempted before login") + if self.auth_mode == "bearer": + headers["Authorization"] = f"Bearer {self.jwt}" + elif self.auth_mode == "jwt-query": + params.append(("jwt", self.jwt)) + # "cookie" mode needs nothing: the CookieJar replays .LexBoxAuth. + + url = self.base_url + path + if params: + url += "?" + urlencode(params) + + data = None + if json_body is not None: + data = json.dumps(json_body).encode("utf-8") + headers["Content-Type"] = "application/json" + elif method == "POST": + data = b"" # POST with an empty body; everything is in the query string + + request = urllib.request.Request(url, data=data, headers=headers, method=method) + try: + with self._opener().open(request, timeout=self.timeout) as response: + return response.status, response.headers, response.read() + except urllib.error.HTTPError as exc: + return exc.code, exc.headers, exc.read() + except urllib.error.URLError as exc: + hint = "" + if isinstance(exc.reason, ssl.SSLError): + hint = ( + " (certificate rejected; self-signed certificates are only accepted " + "for localhost and .localhost subdomains)" + ) + elif self.scheme == "https": + hint = ( + " (a development LexBox serves plain HTTP -- try " + f"--lexbox-host {DEFAULT_LEXBOX_HOST})" + ) + raise Fatal(f"{method} {url} failed: {exc.reason}{hint}") from exc + + @staticmethod + def _describe(status: int, body: bytes) -> str: + """Pull `detail` out of an ASP.NET problem+json response.""" + text = body.decode("utf-8", errors="replace").strip() + try: + document = json.loads(text) + except json.JSONDecodeError: + return f"HTTP {status}: {text[:300]}" + if isinstance(document, dict): + detail = document.get("detail") or document.get("title") + if detail: + return f"HTTP {status}: {detail}" + return f"HTTP {status}: {text[:300]}" + + # -- operations -------------------------------------------------------- # + + def login(self) -> str: + """POST /api/login and return the JWT taken from the .LexBoxAuth cookie.""" + status, _headers, body = self._send( + "POST", + "/api/login", + # defaultScope=true is required: with false the JWT carries no scopes + # and every later call fails the LexboxApi scope requirement. + query=[("defaultScope", "true")], + json_body={ + "password": self.password, + "emailOrUsername": self.username, + "preHashedPassword": False, + }, + ) + if status == 401: + raise Fatal(f"LexBox rejected the credentials for user {self.username!r} (HTTP 401)") + if status != 200: + raise Fatal(f"LexBox login failed: {self._describe(status, body)}") + + for cookie in self.cookie_jar: + if cookie.name == LEXBOX_AUTH_COOKIE: + self.jwt = cookie.value + break + if not self.jwt: + raise Fatal( + f"LexBox login returned HTTP 200 but no {LEXBOX_AUTH_COOKIE} cookie, " + "so no JWT could be obtained." + ) + return self.jwt + + def init_fwdata_project( + self, + code: str, + name: str, + ws_vernacular: list[str], + ws_analysis: list[str], + ) -> str: + """POST /api/project/initFwDataProject and return the new project GUID. + + Multi-valued writing systems are sent as repeated query parameters, which + is how ASP.NET binds `[FromQuery] string[]` (ProjectController.cs:46-47). + The endpoint is admin-only and NOT idempotent, so this never retries. + """ + query: list[tuple[str, str]] = [("code", code)] + query += [("wsVernacular", tag) for tag in ws_vernacular] + query += [("wsAnalysis", tag) for tag in ws_analysis] + if name: + query.append(("name", name)) + + status, _headers, body = self._send( + "POST", "/api/project/initFwDataProject", query=query, authenticate=True + ) + + if status == 200: + # ActionResult serializes as a bare JSON string. + guid = json.loads(body.decode("utf-8")) + if not isinstance(guid, str) or not guid: + raise Fatal(f"initFwDataProject returned an unexpected body: {body[:200]!r}") + return guid + + if status in (401, 403): + raise Fatal( + f"initFwDataProject was refused ({self._describe(status, body)}). " + f"It requires an admin account, and the JWT must be sent in a way LexBox accepts " + f"(current --lexbox-auth={self.auth_mode})." + ) + raise Fatal(f"initFwDataProject failed: {self._describe(status, body)}") + + +def init_lexbox_project( + lexbox: LexboxClient, + lexbox_code: str, + new_name: str, + ws_vernacular: list[str], + ws_analysis: list[str], +) -> str: + """Create the empty LexBox project that the copy will Send/Receive with. + + Logs in and calls initFwDataProject with the writing systems taken from the + Language Forge project config. Returns the GUID it reports. + + `lexbox_code` is the project's sendReceiveProjectIdentifier, which is the + normalised form of the Language Forge projectCode, not the projectCode + itself. + + The project is created empty apart from the template .fwdata: the first + Send/Receive is what pushes the lexicon from Mongo, and the second adds the + assets. + """ + if not ws_vernacular: + raise Fatal( + "LexBox requires at least one vernacular writing system, but the Language Forge " + "project config has none (config.entry.fields.lexeme.inputSystems is empty). " + "Pass --ws-vernacular explicitly." + ) + + if lexbox.scheme != "https": + tls = "not in use (plain HTTP)" + elif lexbox.is_localhost: + tls = "self-signed certificates accepted (localhost family)" + else: + tls = "certificate will be verified" + + log(f" LexBox at {lexbox.base_url}") + log(f" TLS: {tls}") + log(f" logging in as {lexbox.username!r}...") + lexbox.login() + log(f" got JWT from the {LEXBOX_AUTH_COOKIE} cookie ({len(lexbox.jwt or '')} chars)") + + log(f" creating project code={lexbox_code!r} name={new_name!r}") + log(f" wsVernacular: {', '.join(ws_vernacular)}") + log(f" wsAnalysis: {', '.join(ws_analysis) or '(none -- LexBox will default to en)'}") + guid = lexbox.init_fwdata_project(lexbox_code, new_name, ws_vernacular, ws_analysis) + log(f" LexBox project created, GUID {guid}") + + return guid + + +def read_sr_state(config: Config, project_code: str) -> tuple[dict | None, float | None]: + """Read the LFMerge state file. Returns (parsed, mtime), either may be None.""" + path = config.state_file(project_code) + try: + mtime = os.path.getmtime(path) + with open(path, encoding="utf-8") as handle: + raw = handle.read() + except (FileNotFoundError, NotADirectoryError): + return None, None + try: + return json.loads(raw), mtime + except json.JSONDecodeError: + # LFMerge may be part-way through writing it; treat as "not readable yet". + return None, mtime + + +def clear_send_receive_state(config: Config, project_code: str, suffix: str) -> list[str]: + """Remove any webwork clone and LFMerge state file left over from an earlier attempt. + + A LexBox project gets a fresh root changeset every time it is created: initFwDataProject + builds a template .fwdata and commits it, and that commit's hash depends on when it was + made, so creating the same project code twice produces two repositories with no ancestor + in common. A clone taken before the project was recreated is therefore unrelated to it + permanently, and the first Send/Receive fails inside Chorus with + + abort: repository is unrelated + + Nothing else clears it. LFMerge's EnsureCloneAction skips its initial clone whenever the + state file and the .fwdata both exist, so a stale clone is used rather than replaced, and + this script has just created the LexBox project the clone would have to match. + + Only ever called for the copy's code, which this script generates by appending the suffix, + so anything found under it is a leftover from a previous run of this same migration and + never a project a user has data in. The suffix is checked again here because this deletes + a repository. + + Returns the paths removed, for the caller to report. + """ + if not suffix or not project_code.endswith(suffix): + raise Fatal( + f"refusing to clear Send/Receive state for '{project_code}': it does not end with " + f"'{suffix}', so it is not a project code this script generated" + ) + + removed = [] + webwork = config.webwork_path(project_code) + if os.path.isdir(webwork): + shutil.rmtree(webwork) + removed.append(webwork) + + state_file = config.state_file(project_code) + if os.path.exists(state_file): + os.unlink(state_file) + removed.append(state_file) + + return removed + + +def trigger_send_receive(config: Config, project_code: str) -> str: + """Queue the project for Send/Receive by dropping a file in the syncqueue. + + LFMerge's lfmergeqm-looping.sh waits on `inotifywait -e close_write` over + the syncqueue directory, so this fires within milliseconds. The FILENAME is + the project code -- that is what LfMerge's Queue.cs reads; the contents are + ignored, but we write the same line PHP writes + (SendReceiveCommands::queueProjectForSync) for consistency. + """ + queue_dir = config.syncqueue_path + if not os.path.isdir(queue_dir): + os.makedirs(queue_dir, mode=0o2775, exist_ok=True) + + queue_file = config.queue_file(project_code) + with open(queue_file, "w", encoding="utf-8") as handle: + handle.write(f"projectCode: {project_code}") + try: + os.chmod(queue_file, 0o664) + except PermissionError: + pass + chown_to_web_user(queue_file, recursive=False) + return queue_file + + +def wait_for_send_receive(config: Config, project_code: str, *, timeout: int, label: str) -> None: + """Block until LFMerge has finished the queued Send/Receive. + + Two phases, because a stale state file from a previous run would otherwise + look like instant success: + + 1. Pickup -- wait until lfmergeqm has dequeued our file (LfMerge's + Queue.cs deletes it when it starts) AND either the state file has been + rewritten since we queued, or it reports an in-progress state. + 2. Completion -- wait for SRState to reach IDLE/SYNCED, or fail on + HOLD/ERROR. + """ + queue_file = config.queue_file(project_code) + _, state_mtime_at_trigger = read_sr_state(config, project_code) + started = time.monotonic() + picked_up = False + last_reported: str | None = None + + stale, _ = read_sr_state(config, project_code) + stale_state = (stale or {}).get("SRState") + if stale_state in SR_FAILED: + log(f" [{label}] note: previous run left SRState={stale_state}; waiting for a fresh result") + + while True: + elapsed = time.monotonic() - started + queued = os.path.exists(queue_file) + state, mtime = read_sr_state(config, project_code) + srstate = (state or {}).get("SRState") + + if srstate and srstate != last_reported: + log(f" [{label}] SRState={srstate}") + last_reported = srstate + + if not picked_up and not queued: + state_is_fresh = mtime is not None and ( + state_mtime_at_trigger is None or mtime > state_mtime_at_trigger + ) + if srstate in SR_IN_PROGRESS or state_is_fresh: + picked_up = True + + # Both the failure and the success check are gated on pickup, because a + # leftover state file from a previous run would otherwise be read as an + # instant verdict on this one -- an old ERROR would abort a legitimate + # retry, and an old IDLE would declare success before LFMerge started. + if picked_up and srstate in SR_FAILED: + raise Fatal( + f"Send/Receive ({label}) reported SRState={srstate}. " + f"See {config.state_file(project_code)} and `docker compose logs lfmerge`." + ) + + if picked_up and srstate in SR_DONE: + log(f" [{label}] finished after {elapsed:.0f}s (SRState={srstate})") + return + + if not picked_up and elapsed > PICKUP_TIMEOUT_SECONDS: + raise Fatal( + f"LFMerge did not pick up the Send/Receive request within " + f"{PICKUP_TIMEOUT_SECONDS}s (queue file {queue_file} " + f"{'still present' if queued else 'was consumed but no state was written'}). " + "Is the lfmerge container running?" + ) + + if elapsed > timeout: + raise Fatal( + f"Send/Receive ({label}) did not finish within {timeout}s " + f"(last SRState={srstate}). It may still be running; check " + "`docker compose logs lfmerge` before retrying." + ) + + time.sleep(POLL_INTERVAL_SECONDS) + + +def send_receive(config: Config, project_code: str, *, timeout: int, label: str) -> None: + queue_file = trigger_send_receive(config, project_code) + log(f" queued {queue_file}") + wait_for_send_receive(config, project_code, timeout=timeout, label=label) + + +# --------------------------------------------------------------------------- # +# main +# --------------------------------------------------------------------------- # + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Migrate a non-Send/Receive Language Forge project into Send/Receive.", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument("project_code", metavar="projectCode", help="projectCode of the project to copy") + parser.add_argument( + "--suffix", + default=MIGRATE_SUFFIX, + help=f"suffix for the new projectCode (default: {MIGRATE_SUFFIX})", + ) + parser.add_argument( + "--project-name", + default=None, + help="projectName for the copy (default: original name with the suffix appended)", + ) + parser.add_argument( + "--force", + action="store_true", + help="overwrite an existing copy instead of refusing to run", + ) + parser.add_argument( + "--no-assets", + action="store_true", + help="do not copy audio/pictures, and skip the second Send/Receive", + ) + parser.add_argument( + "--no-send-receive", + action="store_true", + help="stop after init_lexbox_project; do not run either Send/Receive", + ) + parser.add_argument( + "--sr-timeout", + type=int, + default=DEFAULT_SR_TIMEOUT_SECONDS, + metavar="SECONDS", + help=f"how long to wait for each Send/Receive (default: {DEFAULT_SR_TIMEOUT_SECONDS})", + ) + + lexbox = parser.add_argument_group("LexBox") + lexbox.add_argument( + "--lexbox-host", + default=DEFAULT_LEXBOX_HOST, + metavar="HOST", + help=( + f"LexBox host (default: {DEFAULT_LEXBOX_HOST}, which is where a development " + "LexBox lives -- it runs outside this Compose environment and is reached via " + "the admin.localhost extra_hosts entry. Production: lexbox.org). HTTPS is used " + "unless a scheme is given; self-signed certificates are accepted only for " + "localhost and .localhost subdomains." + ), + ) + lexbox.add_argument( + "--lexbox-user", + default=DEFAULT_LEXBOX_USER, + metavar="USER", + help=f"LexBox username (default: {DEFAULT_LEXBOX_USER}); must be a LexBox admin", + ) + lexbox.add_argument( + "--lexbox-password", + default=None, + metavar="PASSWORD", + help=( + f"LexBox password (default: the LEXBOX_PASSWORD environment variable, " + f"or {DEFAULT_LEXBOX_PASSWORD!r})" + ), + ) + lexbox.add_argument( + "--lexbox-auth", + choices=("cookie", "jwt-query", "bearer"), + default="cookie", + help=( + "how to present the login JWT on API calls. 'cookie' replays the .LexBoxAuth " + "cookie (default, always works); 'jwt-query' sends ?jwt=; 'bearer' sends " + "an Authorization: Bearer header, which LexBox routes to OpenIddict and rejects " + "whenever OpenId is enabled -- as it is in Development" + ), + ) + lexbox.add_argument( + "--lexbox-timeout", + type=int, + default=DEFAULT_LEXBOX_TIMEOUT_SECONDS, + metavar="SECONDS", + help=f"timeout for LexBox calls (default: {DEFAULT_LEXBOX_TIMEOUT_SECONDS})", + ) + lexbox.add_argument( + "--ws-vernacular", + action="append", + default=None, + metavar="TAG", + help="override the vernacular writing systems (repeat for several, in order)", + ) + lexbox.add_argument( + "--ws-analysis", + action="append", + default=None, + metavar="TAG", + help="override the analysis writing systems (repeat for several, in order)", + ) + parser.add_argument( + "--no-init", + action="store_true", + help=( + "skip the LexBox init and assume the project and its repository already exist, in a " + "correct state. For recovering from a 504 Gateway Timeout on initFwDataProject, which " + "sometimes creates the project anyway. Combines with --force, which only ever touches " + "the Mongo side; under --resume the init is skipped regardless" + ), + ) + parser.add_argument( + "--resume", + action="store_true", + help=( + "finish a migration that stopped because the project has custom fields: skip the copy, " + "the project record and the LexBox init, and run only the Send/Receive steps. Use this " + "once the FLExProject.CustomProperties commit has been pushed to LexBox." + ), + ) + parser.add_argument( + "--custom-fields-only", + action="store_true", + help=( + "report the project's custom fields and stop, without copying anything. Use this to " + "survey what a LexBox custom-field API would have to create for a project." + ), + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="validate and report the plan without changing anything", + ) + return parser.parse_args(argv) + + +def main(argv: list[str]) -> int: + args = parse_args(argv) + config = Config() + + source_code = args.project_code.strip() + if not source_code: + log("ERROR: projectCode must not be empty") + return 2 + if source_code.endswith(args.suffix): + log(f"ERROR: '{source_code}' already ends with '{args.suffix}'; refusing to copy a copy") + return 2 + if args.resume and args.force: + log("ERROR: --resume finishes an existing migration and --force restarts one; pick one") + return 2 + if args.resume and args.no_init: + # Not an error: --resume already skips the init, so --no-init asks for nothing extra. + log("NOTE: --no-init is redundant with --resume, which skips the LexBox init anyway") + + new_code = source_code + args.suffix + # The Send/Receive identifier -- and therefore the LexBox project code -- is + # a normalised form of the projectCode, so the two are not interchangeable. + # Everything LFMerge keys on locally (webwork, state, syncqueue, the project + # database) still uses the projectCode. + new_identifier = send_receive_identifier(new_code) + source_db = database_name(source_code) + target_db = database_name(new_code) + + log(f"Mongo: {config.safe_uri()}") + log(f"Main DB: {config.database}") + + # ---- validate -------------------------------------------------------- # + source = fetch_project(config, source_code) + if source is None: + log(f"ERROR: no project with projectCode '{source_code}' in {config.database}.projects") + return 1 + + app_name = source["appName"] or "lexicon" + new_name = args.project_name if args.project_name is not None else f"{source['projectName']}{args.suffix}" + + log("") + log(f"Source: {source['projectCode']} ({source['projectName']!r}, _id {source['id']})") + log(f" app={app_name} site={source['siteName']} members={source['memberCount']}") + log(f" database={source_db}") + if source["isArchived"]: + log(" NOTE: source project is archived") + + # This script exists to migrate projects INTO the Send/Receive system, so a + # project that already has an identifier is out of scope: copying it would + # produce a second project pointing at a repository that already has a + # different LF project syncing with it. + existing_identifier = (source["sendReceiveProjectIdentifier"] or "").strip() + if existing_identifier: + log("") + log(f"ERROR: '{source_code}' is already a Send/Receive project.") + log(f" sendReceiveProjectIdentifier = {existing_identifier!r}") + log(" This script only migrates projects that are not yet using Send/Receive.") + return 1 + log(" sendReceive: none (eligible for migration)") + + # LexBox will reject the code outright, so check before copying anything. + # Note LF project codes may contain uppercase, which LexBox does not allow. + if len(new_identifier) < LEXBOX_CODE_MIN_LENGTH or not LEXBOX_CODE_RE.match(new_identifier): + log("") + log(f"ERROR: projectCode '{new_code}' normalises to '{new_identifier}',") + log(" which is not a valid LexBox project code.") + log(f" LexBox requires at least {LEXBOX_CODE_MIN_LENGTH} characters matching") + log(f" {LEXBOX_CODE_RE.pattern} (lowercase letters, digits and hyphens only,") + log(" not starting with a hyphen).") + return 1 + + writing_systems = fetch_writing_systems(config, source_code) + # Read from the SOURCE, because this runs before anything is copied. Used to settle which list a + # writing system declared in both should leave. + anchor_counts = count_anchor_writing_system_usage(config, source_db) + ws_vernacular, ws_analysis, ws_unclassified = classify_writing_systems( + writing_systems, args.ws_vernacular, args.ws_analysis, anchor_counts + ) + + log("") + log(f"Copy to: {new_code} ({new_name!r})") + log(f" database={target_db}") + log(f" sendReceiveProjectIdentifier={new_identifier}", ) + if new_identifier != new_code: + log(f" (normalised from projectCode {new_code!r}; LexBox requires lowercase)") + log(f" LexBox project code={new_identifier}") + log(f" webwork={config.webwork_path(new_code)}") + log("") + log("Writing systems (from the Language Forge project config):") + log(f" lexeme={writing_systems['lexeme']} citationForm={writing_systems['citationForm']}") + log(f" definition={writing_systems['definition']} gloss={writing_systems['gloss']}") + log(f" all config fields={writing_systems['config_all']}") + log(f" project.inputSystems={writing_systems['declared']}") + if anchor_counts: + summary = ", ".join( + f"{tag}={counts.get('vernacular', 0)}v/{counts.get('analysis', 0)}a" + for tag, counts in sorted(anchor_counts.items()) + ) + log(f" entries with data (vernacular/analysis fields): {summary}") + + if ws_unclassified: + log("") + log("WARNING: these writing systems are in the project but are neither vernacular") + log(f" nor analysis by the field conventions: {', '.join(ws_unclassified)}") + log(" Treating them as ANALYSIS writing systems so that they still reach the") + log(" FieldWorks project. Check the result, and pass --ws-vernacular /") + log(" --ws-analysis if any of them belong somewhere else.") + log("") + + source_note = " (from --ws-vernacular)" if args.ws_vernacular else "" + log(f" wsVernacular: {ws_vernacular}{source_note}") + notes = [] + if args.ws_analysis: + notes.append("from --ws-analysis") + if ws_unclassified: + notes.append(f"includes {len(ws_unclassified)} unclassified") + source_note = f" ({', '.join(notes)})" if notes else "" + log(f" wsAnalysis: {ws_analysis}{source_note}") + if not ws_vernacular: + log("") + log("ERROR: LexBox requires at least one vernacular writing system, but this project's") + log(" config.entry.fields.lexeme.inputSystems is empty. Pass --ws-vernacular.") + return 1 + log("") + + # Custom fields are read from the SOURCE project: this runs before anything is copied, so the + # target database does not exist yet. Report only -- LfMerge cannot create custom fields in + # FieldWorks ("Custom field X from LF skipped, because we're not yet creating new custom fields + # in LCM"), so anything listed here will not survive the migration until something does. + custom_fields = describe_custom_fields( + fetch_custom_fields(config, source_code), + scan_custom_field_usage(config, source_db), + ) + report_custom_fields(custom_fields) + log("") + + # Two custom fields whose names differ only by case have to be separated before FieldWorks sees + # them. Checked here, before anything has been copied, so an unhandled clash costs nothing. + custom_field_renames, rename_problems = plan_custom_field_renames(custom_fields) + if custom_field_renames or rename_problems: + log("Custom field names differing only by case:") + report_custom_field_renames(custom_field_renames, rename_problems) + log("") + if rename_problems: + log("ERROR: refusing to migrate a project whose custom field names differ only by case") + log(" without knowing which one should keep its name. FLEx is inconsistent about") + log(" whether it treats such a pair as one field or two, so migrating them as they") + log(" are risks conflating two fields that hold different data.") + log(f" Add the name to rename to CASE_CLASH_CUSTOM_FIELD_RENAMES in " + f"{os.path.basename(sys.argv[0])}, having first checked which of the two should") + log(" keep its name and whether they hold the same kind of data.") + return 1 + # Everything downstream -- the FieldWorks definitions, and so the XML -- has to describe the + # fields under the names the copy will actually use. + apply_custom_field_renames(custom_fields, custom_field_renames) + + # LfMerge skips any LF custom field that does not already exist in FieldWorks, and the first + # Send/Receive is the only pass that would have carried the data. So a project with custom + # fields stops before Send/Receive and prints the definitions for a human to commit; --resume + # picks it up again afterwards. + custom_field_defs, custom_fields_dropped = plan_custom_field_definitions( + custom_fields, ws_vernacular, ws_analysis + ) + stop_for_custom_fields = bool(custom_field_defs) and not args.resume + + if args.custom_fields_only: + # Show the definitions this project would need, but write nothing: this flag exists to + # survey a project without touching it, and a file on disk is a change to something. + if custom_field_defs: + report_custom_field_definitions(custom_field_defs, custom_fields_dropped) + log_custom_properties_xml(build_custom_properties_xml(custom_field_defs)) + log("") + log("--custom-fields-only: stopping here, nothing was changed and no file was written.") + return 0 + + existing = fetch_project(config, new_code) + target_collections = count_collections(config, target_db) + if args.resume: + # --resume continues a run that stopped for custom fields, so the copy it is resuming has + # to still be there. Recreating it here would give the project a new LexBox repository and + # throw away the custom field commit that --resume exists to wait for. + if existing is None: + log(f"ERROR: --resume: there is no project '{new_code}' to resume.") + log(" Run without --resume first; that is what creates the copy.") + return 1 + if target_collections == 0: + log(f"ERROR: --resume: database '{target_db}' is empty, so the copy is not there to resume.") + return 1 + log(f"--resume: continuing the existing copy '{new_code}' (_id {existing['id']})") + log("") + else: + if existing is not None and not args.force: + log(f"ERROR: project '{new_code}' already exists (_id {existing['id']}). Use --force to overwrite.") + return 1 + if target_collections > 0 and not args.force: + log(f"ERROR: database '{target_db}' already has {target_collections} collection(s). Use --force to overwrite.") + return 1 + + special = SPECIAL_LEXICON_INT_REFERENCES.get(source_code) + if special: + special_field, special_values = special + log(f"Special case: {source_code!r} has a lexicon reference to an option list item") + log(f" by number ({special_field}.values == {special_values}). That sense will be") + log(" coerced to strings along with the option list keys.") + log("") + + if args.dry_run: + log("Planned sequence:") + log(" 0. (already done above) report the source project's custom fields") + if args.resume: + log(" --resume: steps 1-6 are SKIPPED; the existing copy is picked up at step 7") + log(f" 1. copy database {source_db} -> {target_db}") + log(f" 2. coerce numeric option list keys in {target_db} to strings" + + (", and the special-case lexicon sense" if special else "")) + empty_guid_count = len(find_empty_entry_guids(config, source_db)) + log(f" 3. replace empty entry guids with fresh UUID4s" + f" ({empty_guid_count} found in {source_db})") + if custom_field_renames: + log(f" 3b. rename {len(custom_field_renames)} custom field(s) in {target_db}.lexicon" + " and in the copied config:") + for rename in custom_field_renames: + log(f" {rename['level']:8s} {rename['old_key']} -> {rename['new_key']}") + log(" 4. stage assets" + (" (SKIPPED: --no-assets)" if args.no_assets else "")) + log(" 5. create project record") + if args.no_init: + log(f" 6. SKIPPED: --no-init; assuming LexBox project {new_identifier!r} already exists") + else: + log(f" 6. init_lexbox_project: log in to {split_lexbox_host(args.lexbox_host)[0]}://" + f"{split_lexbox_host(args.lexbox_host)[1]} as {args.lexbox_user!r},") + log(" POST /api/project/initFwDataProject") + if args.no_send_receive: + log(" 7-9. SKIPPED: --no-send-receive") + elif stop_for_custom_fields: + log(f" 7-9. SKIPPED: {len(custom_field_defs)} custom field(s) found. The run will stop after") + log(f" step 6 and write {CUSTOM_PROPERTIES_FILENAME} for a human to commit to the") + log(" project's Mercurial repository, then be resumed with --resume.") + else: + log(f" 7. clear any stale webwork clone and state file for {new_code},") + log(" then Send/Receive #1 (push Mongo data), wait for completion") + if args.no_assets: + log(" 8-9. SKIPPED: --no-assets") + else: + log(" 8. move assets into webwork LinkedFiles, symlink back") + log(" 9. Send/Receive #2 (push assets), wait for completion") + log("") + log("--dry-run: stopping here, nothing was changed.") + return 0 + + # ---- copy ------------------------------------------------------------ # + # Undo actions, applied in reverse order if a later step fails. Cleared once + # the first Send/Receive starts, since by then data has left for LexBox and + # LFMerge owns the webwork tree. + undo: list[tuple[str, Callable[[], None]]] = [] + new_id = None + lexbox_guid = None + coercion: dict | None = None + lexicon_coercion: dict | None = None + guid_fix: dict | None = None + rename_result: dict | None = None + + try: + if args.resume: + # Everything below was done by the run that stopped for custom fields. Redoing any + # of it would replace the LexBox project, and with it the repository holding the + # custom field commit that --resume exists to wait for. + new_id = existing["id"] + log("--resume: skipping the copy, the project record and the LexBox init") + else: + if args.force and existing is not None: + log(f"--force: removing existing project '{new_code}' (_id {existing['id']})") + delete_project_record(config, existing["id"]) + + log(f"Copying database {source_db} -> {target_db}") + copy_project_database(config, source_db, target_db, drop=args.force) + undo.append( + ( + f"drop database {target_db}", + lambda: mongosh_json( + config, + f"db.getSiblingDB({json.dumps(target_db)}).dropDatabase();\nprint(JSON.stringify({{ok:1}}));\n", + what=f"Dropping {target_db}", + ), + ) + ) + + # Before anything else looks at the copy: option list keys are meant to + # be strings, and the lexicon already references them that way. Runs + # inside the same try, so a failure here rolls the copy back. + log(f"Coercing numeric option list keys in {target_db}") + coercion = coerce_option_list_keys(config, target_db) + lexicon_coercion = coerce_special_lexicon_references(config, target_db, source_code) + + # Also before the first Send/Receive: an entry with an empty guid would + # be pushed to FieldWorks under the all-zeros GUID. + log(f"Replacing empty entry guids in {target_db}") + guid_fix = fix_empty_entry_guids(config, target_db) + + if custom_field_renames: + log(f"Renaming custom fields in {target_db}") + rename_result = rename_custom_fields_in_lexicon( + config, target_db, custom_field_renames + ) + + if args.no_assets: + log("Skipping assets (--no-assets)") + else: + log("Staging assets") + staged = stage_assets(config, app_name, source_db, target_db) + if staged: + undo.append( + ( + f"remove assets {staged}", + lambda path=staged: shutil.rmtree(path, ignore_errors=True), + ) + ) + + log("Creating project record") + new_id = create_project_record( + config, source_code, new_code, new_name, new_identifier, custom_field_renames + ) + undo.append( + (f"remove project record {new_id}", lambda pid=new_id: delete_project_record(config, pid)) + ) + + if args.no_init: + # Nothing here is rolled back on failure -- init_lexbox_project has no undo entry, + # which is exactly how a 504 leaves a LexBox project behind while everything on the + # Language Forge side is undone. Re-creating it would fail on the existing code, so + # this trusts the project that is already there. + log(f"Skipping LexBox init (--no-init): assuming project {new_identifier!r} and its") + log(f" repository already exist at {args.lexbox_host}, with the right writing systems.") + log(f" Expected wsVernacular: {', '.join(ws_vernacular)}") + log(f" Expected wsAnalysis: {', '.join(ws_analysis) or '(none)'}") + else: + log("Initializing LexBox project") + lexbox_client = LexboxClient( + args.lexbox_host, + args.lexbox_user, + args.lexbox_password + if args.lexbox_password is not None + else os.environ.get("LEXBOX_PASSWORD", DEFAULT_LEXBOX_PASSWORD), + auth_mode=args.lexbox_auth, + timeout=args.lexbox_timeout, + ) + lexbox_guid = init_lexbox_project( + lexbox_client, + new_identifier, + new_name, + ws_vernacular, + ws_analysis, + ) + + if stop_for_custom_fields: + # Deliberately before any Send/Receive: LfMerge would skip every one of these fields + # and the data in them, and the first Send/Receive is the only pass that would have + # carried it. The definitions have to reach the repository first. + custom_properties_path = emit_custom_properties( + custom_field_defs, custom_fields_dropped, new_code, new_identifier, args.lexbox_host + ) + log("") + log("Send/Receive NOT run, and no data has been pushed to LexBox yet.") + log(f"Once {custom_properties_path} has been committed and pushed, finish with:") + log(f" {os.path.basename(sys.argv[0])} {source_code} --suffix {args.suffix} --resume") + elif args.no_send_receive: + log("Skipping both Send/Receive runs (--no-send-receive)") + log(" NOTE: assets are still real directories; they will not be linked into") + log(" webwork until a Send/Receive has created it.") + else: + # A newly created LexBox project has a root changeset no earlier clone can share, so + # any clone a previous attempt left behind has to go, or LFMerge would reuse it and + # Chorus would abort the pull as unrelated. Done under --no-init as well: the clone is + # then of the same repository and would not be rejected, but a clone left by an attempt + # that failed partway is not worth trusting either. + stale = clear_send_receive_state(config, new_code, args.suffix) + for path in stale: + log(f"Removed stale Send/Receive state from an earlier attempt: {path}") + + # Past this point a failure is not safely reversible: LFMerge has + # started pushing to LexBox and owns the webwork clone. + undo.clear() + + log("Send/Receive #1: pushing Mongo data to LexBox") + send_receive(config, new_code, timeout=args.sr_timeout, label="1/2") + + if args.no_assets: + log("Skipping asset linking and Send/Receive #2 (--no-assets)") + else: + log("Moving assets into the Send/Receive work tree") + link_assets_into_webwork(config, app_name, target_db, new_code) + + log("Send/Receive #2: pushing assets to LexBox") + send_receive(config, new_code, timeout=args.sr_timeout, label="2/2") + + except Fatal as exc: + log("") + log(f"ERROR: {exc}") + if undo: + log("Rolling back:") + for description, action in reversed(undo): + log(f" {description}") + try: + action() + except Exception as cleanup_exc: # noqa: BLE001 - best effort + log(f" rollback step failed: {cleanup_exc}") + elif new_id is not None: + log("NOT rolling back: Send/Receive has already run, so the copy and its") + log(f"LexBox project may hold real data. Inspect '{new_code}' before retrying.") + return 1 + + # ---- report ---------------------------------------------------------- # + log("") + if stop_for_custom_fields: + log(f"Paused for custom fields; not finished. '{source_code}' was left untouched.") + else: + log(f"Done. '{source_code}' was left untouched.") + log(f" new project: {new_code} (_id {new_id})") + log(f" new database: {target_db}") + if coercion: + log(f" option list keys coerced to strings: {coercion['keysCoerced']}" + f" (in {coercion['listsChanged']} list(s))") + if lexicon_coercion and lexicon_coercion["valuesCoerced"]: + log(f" lexicon values coerced to strings: {lexicon_coercion['valuesCoerced']}" + f" (in {lexicon_coercion['entriesChanged']} entry/entries)") + if guid_fix and guid_fix["fixed"]: + log(f" empty entry guids replaced: {guid_fix['fixed']}") + if rename_result and rename_result["keysRenamed"]: + log(f" custom field keys renamed: {rename_result['keysRenamed']}" + f" (in {rename_result['entriesChanged']} entry/entries)") + for rename in custom_field_renames: + log(f" {rename['old_name']!r} -> {rename['new_name']!r}") + if new_identifier != new_code: + log(f" S/R identifier: {new_identifier}") + if lexbox_guid: + log(f" LexBox GUID: {lexbox_guid}") + elif args.no_init and not args.resume: + log(f" LexBox: init skipped (--no-init); used the existing {new_identifier!r}") + if not args.no_send_receive and not stop_for_custom_fields: + log(f" state file: {config.state_file(new_code)}") + if not args.no_assets: + log(f" assets now symlink into {config.linked_files_path(new_code)}") + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main(sys.argv[1:])) + except Fatal as exc: + log(f"ERROR: {exc}") + sys.exit(1) + except KeyboardInterrupt: + log("Interrupted.") + sys.exit(130)