diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 72f2ba3..a5e7aa5 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -193,15 +193,22 @@ jobs: # The claims the pages make in prose — the counts, the completeness, the rules — # which a table cannot carry and a name cannot say. - - name: The claims the pages make + # The claims the pages make, and the client the skill carries: the client is a + # program that talks to an application over a socket, and nothing else would say + # whether it asks what it should. + - name: The claims the pages make, and the client the skill carries working-directory: documentation run: vendor/bin/pest # A site whose links 404 is worse than no site, and nothing else says so: the build # succeeds, the pages are there, and the reader is the one who finds out. - - name: The site's own links + # + # The skill is checked here too, because it is built into the site: a bundle that has + # fallen behind the pages it was made from is a bundle that is quietly wrong, and + # nothing else would say so. + - name: The site's own links, and the skill it publishes working-directory: documentation - run: php tools/check-site-links.php + run: npm run check:site # The pages under `content/command-line/` are written by hand, and nothing else says # whether they still describe the program. A command that is implemented and not diff --git a/README.md b/README.md index ab7d893..d123abf 100644 --- a/README.md +++ b/README.md @@ -197,6 +197,18 @@ Those are the site's own routes. The site is served at the root of its host, so files carry no path in front of them: the addresses in `llms.txt` and `pages.json` are absolute, and the links inside the Markdown mirrors are the routes as they are written. +## The skill + +The site also publishes a **skill**: `/skills/mdword.zip`, and `/skills/mdword/` file by +file, with `/skills/index.json` as the catalogue OpenCode reads over the network. It is the +documentation above as a directory an agent can read, with the examples beside it and the +tools that render a document so that it can be looked at. + +**It is built by the same build**, out of the same sources, so it cannot say anything the site +does not say. `tools/check-skill.php` is what holds it to that: every reference file must be +the page it carries, every page of the sections it covers must be carried, and the scripts +must be the ones the library ships. + ## Where it is published `.github/workflows/pages.yml` builds the site and publishes it to diff --git a/composer.json b/composer.json index 63e1a17..67da845 100644 --- a/composer.json +++ b/composer.json @@ -84,7 +84,7 @@ "@php tools/check-readme-links.php ../markdown-word ../markdown-presentation ../mdoffice", "@test" ], - "check:site": "@php tools/check-site-links.php", + "check:site": ["@php tools/check-site-links.php", "@php tools/check-skill.php"], "check:links": "@php tools/check-readme-links.php ../markdown-word ../markdown-presentation ../mdoffice" }, "minimum-stability": "dev", diff --git a/content/skills/index.md b/content/skills/index.md new file mode 100644 index 0000000..8b80e22 --- /dev/null +++ b/content/skills/index.md @@ -0,0 +1,100 @@ +--- +layout: layouts/base.njk +permalink: /skills/ +title: The skill +summary: A directory an agent reads, holding this site's documentation and the library's examples — and where each agent harness looks for one. +section: The skill +order: 0 +--- + +An agent asked to turn Markdown into a Word document has to know four things: what the +library can do, how to build a document with it, how to *look* at what it built, and how to +use a design that already exists. This page is how it gets them. + +## What it is + +**A skill is a directory.** It holds a `SKILL.md` — a short account of what it is for and +when to use it — and beside it the files it points at: the documentation, the examples, and +the scripts that render a document so that it can be looked at. + +**The one for this suite is `mdword`**, and it is built from this site. The pages in it are +the pages of this site; the examples in it are this site's own and the library's. Nothing in +it is written twice: it is a copy made by the build, and a copy that is made by the build +cannot drift from what it was made from. + +## Install it + +OpenCode reads it over the network and needs nothing downloaded — one line in +`opencode.json`: + +```jsonc +{ + "$schema": "https://opencode.ai/config.json", + "skills": ["https://markdown-office.github.io/skills/"] +} +``` + +Every other agent reads a directory, and the whole bundle is one file: + +```bash +curl -LO https://markdown-office.github.io/skills/mdword.zip +unzip -q mdword.zip -d ~/.agents/skills/mdword +``` + +## Where it goes + +| Agent | It looks here | What to do | +| --- | --- | --- | +| **Codex** | `~/.agents/skills/mdword` | nothing — that is where it is | +| **OpenCode** | `~/.agents/skills/mdword`, `~/.config/opencode/skills/mdword` and `~/.claude/skills/mdword` | nothing, or the catalog above | +| **Claude Code** | `~/.claude/skills/mdword` | a symlink to the unpacked copy | +| **GitHub Copilot** | `~/.copilot/skills/mdword`, and `.github/skills/mdword` in a repository | a symlink to the unpacked copy | + +So the bundle is unpacked in one place and the other two are links to it: + +```bash +ln -s ~/.agents/skills/mdword ~/.claude/skills/mdword +ln -s ~/.agents/skills/mdword ~/.copilot/skills/mdword +``` + +A repository can carry one of its own instead, under `.agents/skills/` or `.github/skills/`, +and it is found from the repository root rather than from the machine. + +## Is the copy current? + +The version is in `VERSION` beside the `SKILL.md`, and it is the same string as the `version` +in the published catalogue. A copy whose `VERSION` differs from the catalogue's is a copy +from before, and the catalogue is the newer of the two: + +```bash +curl -s https://markdown-office.github.io/skills/index.json +``` + +## What is in it + +``` +mdword/ +├── SKILL.md what the library does, and how to use it +├── VERSION the version of this bundle +├── references/ the documentation, one file per page +├── examples/ this site's examples, and the library's +│ ├── site/ +│ └── library/ +└── scripts/ the render tools +``` + +The **references** are the pages of this site, one file each, with the frontmatter off. The +**examples** are the decks and documents the pages show, built and named, and the library's +own twenty-three — which between them are the catalogue of what the library can do. + +The **scripts** render a document so that it can be looked at, which is the one step a +prose answer cannot do for itself. + +## What it is not + +**It does not replace this documentation.** It is a copy of it, made by the same build that +makes the site, and the site is where a change is made first. A copy of the documentation is +not the documentation: it is a copy of it, as of the moment the copy was made. + +And it is **not a second implementation of anything**. The library is the library; the skill +says how to drive it. diff --git a/content/word/errors.md b/content/word/errors.md index 4d4b8d3..749205e 100644 --- a/content/word/errors.md +++ b/content/word/errors.md @@ -4,7 +4,7 @@ permalink: /word/errors/ title: Errors summary: Every exception Markdown Word throws, and what each one means. section: Markdown Word -order: 5 +order: 6 --- Every exception Markdown Word throws, in one place. The one to catch, and the advice that is diff --git a/content/word/images.md b/content/word/images.md index cf20d65..8b17528 100644 --- a/content/word/images.md +++ b/content/word/images.md @@ -4,7 +4,7 @@ permalink: /word/images/ title: Image options summary: The three options that decide what happens to a picture, and the two calls that report what happened to it. section: Markdown Word -order: 4 +order: 5 --- Word holds a picture one of three ways, and the option says which. The formats themselves — what diff --git a/content/word/index.md b/content/word/index.md index db32f3f..fb502f2 100644 --- a/content/word/index.md +++ b/content/word/index.md @@ -281,10 +281,18 @@ try { The whole catalogue is on [Errors](/word/errors/). +### For an agent + +There is a **skill**: a directory an agent reads, holding this half's documentation, the +library's examples, and the tools that render a document so that it can be looked at. It is +built from this site, so it is never out of step with it. + +[How to install it](/skills/). + --- ## Next [Markdown Presentation](/presentation/) · [The directive syntax](/directives/syntax/) · [Word directives](/word/directives/) · [Images](/images/) · -[Round trips](/word/round-trip/) · [Errors](/word/errors/) \ No newline at end of file +[Round trips](/word/round-trip/) · [Errors](/word/errors/) · [The skill](/skills/) \ No newline at end of file diff --git a/content/word/round-trip.md b/content/word/round-trip.md index 980ef28..675d9de 100644 --- a/content/word/round-trip.md +++ b/content/word/round-trip.md @@ -4,7 +4,7 @@ permalink: /word/round-trip/ title: Round trips summary: Markdown to a document and back. What comes out the same, and the six things that do not. section: Markdown Word -order: 3 +order: 4 --- Both directions are the same interface, so a round trip is two calls: diff --git a/content/word/templates.md b/content/word/templates.md new file mode 100644 index 0000000..fc37155 --- /dev/null +++ b/content/word/templates.md @@ -0,0 +1,170 @@ +--- +layout: layouts/base.njk +permalink: /word/templates/ +title: Templates +summary: Rendering into a design that already exists — a Word template with its own styles, and a theme for one that is not a Word document. +section: Markdown Word +order: 3 +--- + +## Two ways to bring a design + +There are two, and they answer different questions. + +| | The design is | You give it | It gives you | +| --- | --- | --- | --- | +| **A template** | a Word document — a `.docx` or a `.dotx` | the `styleId`s of its styles | its fonts, colours, spacing, headers and footers, and its letterhead | +| **A theme** | a directory of YAML | nothing | a look of your own, in every document that names it | + +A template is a design somebody made in Word. A theme is a design you write down, and it is +the only one of the two that travels with the Markdown rather than with the document. + +## A Word template + +Point the renderer at the **styleIds** your template defines and it inherits the whole design +— fonts, colours, spacing, headers and footers. + +```php +use MarkdownWord\Configuration; +use MarkdownWord\Configuration\Styles; +use MarkdownWord\Template\MarkdownTemplate; + +$config = Configuration::create()->withStyles([ + Styles::HEADING_1 => 'ReportTitle', // a styleId in your template + Styles::PARAGRAPH => 'BodyText', + Styles::BLOCK_QUOTE => 'PullQuote', +]); + +(new MarkdownTemplate('report-template.docx', $config)) + ->insert(file_get_contents('summary.md')) + ->save('northwind.docx'); +``` + +A `.docx` and a `.dotx` are both accepted, and which one it is makes no difference to the +result: **a filled-in template is always a document.** See [What a `.dotx` costs](#what-a-dotx-costs) +for why that is worth saying. + +Everything a template says is something Word already understands, because a template comes from +Word and anything else would have to be invented here: + +| | | +| --- | --- | +| A value | a **`MERGEFIELD`**, the field a person inserts with `Ctrl+F9` | +| A place for the content | a **rich-text content control**, named `body` | + +There is no third thing. A row of records is a table in the Markdown, and the template styles +it like any other content — which is why there is no loop to explain. + +### The values + +`fields()` fills the template's `MERGEFIELD`s, and the document carries its own values so +that the document and everything in it travel together: + +```php +(new MarkdownTemplate('report-template.docx')) + ->fields(['customer' => 'Northwind Ltd']) + ->insert(file_get_contents('summary.md')) + ->save('northwind.docx'); +``` + +**Filled fields are unlinked.** The value stays and the field goes, which is what Word's own +*Finish & Merge → Edit Individual Documents* produces and what a finished document wants: +nothing to refresh with `F9`, and a value that can hold `&` or `<` without becoming an +instruction. + +### The place for the content + +**`body` is this library's convention, not Word's.** ECMA-376 §17.5.2.36 defines a content +control's `w:tag` as "an arbitrary string which applications can associate with a structured +document tag", reserves no value for it, and Word itself never reads one — it round-trips it. +So the name collides with nothing Word recognises, and no standard is being missed by not +matching one. + +The control is matched on its **tag**, which is the field name Word gives it, rather than on +its alias: the alias is the label a person sees in Word's own UI and is the one they will +rename while editing the template, whereas the tag is what a program reads. A template that +has both is written by something that knows the difference. + +A template that names its control something else is asked for that name: + +```php +(new MarkdownTemplate('letterhead.docx'))->insert($markdown, 'letter'); +``` + +Nobody else does this: the templating libraries that exist either use plain text delimiters +(`{name}` in docxtemplater, `{{ var }}` in python-docx-template) or invent their own prefix, +and none of them claims `body`. + +### A template with no control + +**A template need not be one this library's generator wrote.** A letterhead a company made in +Word to be filled in by a person has fields and no control, and refusing those would leave +the library able to use only the templates it wrote itself. + +So a template with no control of that name takes the content **at the end of its body**, or +says so and refuses. Which one is the `contentFallback()` setting — `ContentFallback::Append` +or `ContentFallback::Error` — and the refusal is `--require-control` on the command line. + +Filling the fields is independent of it: a template can have values and nowhere to put the +prose. + +## The same thing from the frontmatter + +A document may name its own template, and then nothing has to be supplied from PHP to +reproduce it: + +```yaml +--- +template_file: ../templates/report.dotx +fields: + customer: Northwind Ltd +--- +``` + +`template_file:` names a `.docx` or a `.dotx`, and the path is resolved against the +document's own directory. `fields:` fills the template's `MERGEFIELD`s, with `true` and +`false` spelled as strings because that is what Word stores. + +The two routes are the same thing reached two ways, and the difference is where the styleIds +come from: the frontmatter route cannot name them, so it inherits the template's own styles +rather than mapping this library's slots onto them. + +## What a `.dotx` costs + +Nothing to use one, and one thing to *produce* from one: the result is a document, not a +filled template. That is the only difference between the two, and it is worth saying because +the alternative is worse. + +A `.dotx` and a `.docx` are the same archive with one line of `[Content_Types].xml` different, +and that line is what Word decides a file by. A package typed as a template is opened by +*creating a new document from it* — double-clicking a filled-in `.dotx` gives you +`Document1.docx` rather than the file you were sent, and anything dispatching on MIME type +treats it as a template. So rendering into one writes a document, and says so by writing one: + +```php +$template = new MarkdownTemplate('report-template.dotx'); +$template->insert('body', $markdown)->save('northwind.docx'); // a .docx, whatever went in +``` + +## A theme + +A theme is a directory: `theme.yml` — `name` required, plus `description` and `author` — and +an optional `styles.yml` holding the same slot names as +[the configuration](/word/configuration/#every-style-slot). One key of that file is not a slot: +`textStyles` holds the named styles a document may write a word in. + +A document names one with `theme:`, and the ones that ship are listed with the configuration, on +[the themes](/word/configuration/#themes). + +A theme may also name the `theme.yml` itself, in which case the `styles.yml` beside it is the +theme's, and a document that names one by path gets that one. + +## The command line + +```bash +mdoffice word to-docx notes.md --template report.dotx --require-control +``` + +`--template` takes a `.docx` or a `.dotx`, and `--require-control` refuses a template with no +content control instead of appending to it. The rest of the options are on +[For word](/command-line/word/). diff --git a/eleventy.config.js b/eleventy.config.js index 24636c3..9d9e087 100644 --- a/eleventy.config.js +++ b/eleventy.config.js @@ -1,19 +1,13 @@ -import { createHash } from "node:crypto"; import { mkdir, readFile, writeFile } from "node:fs/promises"; import { existsSync, readFileSync } from "node:fs"; import { fileURLToPath } from "node:url"; import { dirname, join } from "node:path"; import { EleventyHtmlBasePlugin } from "@11ty/eleventy"; -import { createRequire } from "node:module"; import { createHighlighter } from "shiki"; -// js-yaml has moved between ESM and CommonMark across majors; going through `createRequire` -// loads whichever it currently is rather than betting on an import shape. -const yaml = createRequire(import.meta.url)("js-yaml"); - -// js-yaml renamed `parse` to `load` in v5 and moved between module systems on the way, so the -// two names are both asked for rather than betting on which one this project has installed. -const parseYaml = yaml.parse ?? yaml.load; +import { asMarkdown, hashOf } from "./tools/markdown.js"; +import { parseYaml } from "./tools/yaml.js"; +import { writeSkill } from "./tools/write-skill.js"; const here = dirname(fileURLToPath(import.meta.url)); @@ -52,6 +46,7 @@ const SECTIONS = [ ["Markdown Presentation", "The dialect, the directives, the themes, and the errors."], ["In general", "The parts that are about both libraries."], ["The command line", "One command over both of them."], + ["The skill", "What an agent can install, and where it goes."], ["The suite", "The other packages, and where this is going."], ]; @@ -116,6 +111,7 @@ export default function (eleventyConfig) { eleventyConfig.on("eleventy.after", highlight); eleventyConfig.on("eleventy.after", writeMirrors); + eleventyConfig.on("eleventy.after", writeSkill); return { dir: { @@ -448,19 +444,6 @@ function download(href, label, about, suffix) { ); } -/** - * The key a built example is filed under, and it has to be the same on both sides of the build. - * - * `tools/render-examples.php` computes it as the first sixteen hex digits of - * `hash('sha256', $block)`, where `$block` is the whole trimmed fenced block — opening fence, body - * and closing fence. Hashing the body alone would need the body recovered from the text Eleventy - * hands over, which is the whole block; a fence is the one thing the shortcode cannot strip - * without re-implementing the parser that produced it. - */ -function hashOf(block) { - return createHash("sha256").update(block, "utf8").digest("hex").slice(0, 16); -} - function readManifest() { try { return JSON.parse(readFileSync(join(here, "_data/examples.json"), "utf8")); @@ -659,23 +642,6 @@ function frontmatterOf(data) { }; } -/** - * A page's body with its frontmatter off and its examples in the form a reader can paste. - * - * The `{% example %}` pair is the build's own machinery, and this file is not the build — so an - * example becomes the fence it was written as. That is the whole point of including them: a - * reader gets the document source, ready to use, rather than a reference to a picture of one. - */ -function asMarkdown(source) { - const [, , body = ""] = - source.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n?([\s\S]*)$/) ?? []; - - return body - .replace(/\{%\s*example[^%]*%\}\n?/g, "") - .replace(/\n?\{%\s*endexample\s*%\}/g, "") - .trim(); -} - /** * The pages grouped for the sidebar, and `Elsewhere` for anything in a section the menu does not * declare. Listing those rather than dropping them turns a naming mistake into a visible page diff --git a/phpunit.xml.dist b/phpunit.xml.dist index adfc253..80dda1b 100644 --- a/phpunit.xml.dist +++ b/phpunit.xml.dist @@ -15,5 +15,8 @@ tests/Claims + + tests/Skill + diff --git a/skill/mdword/SKILL.md b/skill/mdword/SKILL.md new file mode 100644 index 0000000..db1367a --- /dev/null +++ b/skill/mdword/SKILL.md @@ -0,0 +1,96 @@ +--- +name: mdword +description: Turn Markdown into a Word document — .docx, .odt or .rtf — with the markdown-word library, and turn one back into Markdown. Use when asked to write or convert a Word document, to render Markdown into a company's own Word template or theme, to read a Word document back into Markdown, or to check that a document the library produced looks right. +--- + +# mdword + +Markdown becomes a Word document with `markdown-word`. **The answers are in the files +below, and the answers are the library's own** — this file says where to look and nothing +else. A question about what the library can do is answered by reading the library's +documentation, not by recalling it. + +## Where to look + +| The question | The file | +| --- | --- | +| The dialect — headings, tables, lists, code, links, pictures — and the library's own API | `references/dialect.md` | +| The directives: `::toc`, `:footnote`, `:style`, and the seven that place objects in a document | `references/directives.md` | +| Every frontmatter key, every option and every style slot | `references/configuration.md` | +| Rendering into a design that already exists — a Word template, or a theme | `references/templates.md` | +| What comes back when a document is read into Markdown, and the six things that do not | `references/round-trip.md` | +| The three options that decide what happens to a picture | `references/images.md` | +| The same three options, and how the slides half differs | `references/general-images.md` | +| Every exception the library throws, and what each one means | `references/errors.md` | +| How a directive is written — names, labels, attributes, fences | `references/directive-syntax.md` | +| What the two halves of the suite do with a name they know, and with one they do not | `references/directive-rules.md` | +| The errors both halves throw | `references/errors-general.md` | +| The command line, and the options each command takes | `references/command-line-word.md`, `references/command-line-basic.md` | +| Installing the command line | `references/command-line-install.md` | +| Installing this skill | `references/install.md` | + +The **examples** are beside these: `examples/site/` holds the documents this documentation +shows, and `examples/library/` holds the library's own twenty-three — the catalogue of what +it can do, one file each. + +## The loop + +Write the Markdown, build the document, render it, **look at it**, and fix the Markdown. + +```bash +mdoffice word to-docx notes.md -o notes.docx # build +php scripts/render-document.php notes.docx --out notes.pdf # render +``` + +A **template** is rendered as well as a document — `php scripts/render-document.php +report.dotx --out report.pdf` — and the PDF is named after the template. + +**Then read the PDF.** A document that has not been looked at has not been checked, and the +thing a document gets wrong is almost always something only a picture shows: a line that +wraps, a table that runs off the page, a picture in the wrong place. + +For a picture of every page rather than a PDF, WordCraft renders them directly: + +```bash +php scripts/render.php notes.docx --port 7981 --out pages/ +``` + +The port is the application's own — start it with `--control `, and read the key it +writes out of its settings folder. The script finds the key itself. + +## Which renderer to believe + +| | What it is | How much it can be trusted | +| --- | --- | --- | +| **Microsoft Word** | the real thing | **the truth** — the only one that will say whether a document opens | +| **WordCraft** | a clean-room Word | faithful, and the only one of the three that runs everywhere | +| **LibreOffice** | headless | approximate — it disagrees with Word about where a line falls often enough that it cannot answer the question a renderer is asked | + +`scripts/render-document.php` picks the first of them the machine has, and **says so when +it had to fall back**, because a renderer that cannot be trusted is worse than no renderer: +a caller who does not know is told a layout is right when it is not. + +## What this half does differently + +**The Word half never refuses.** A directive it does not know is written into the document +as the author's own characters — `Status :green[go] before lunch.` reaches the document +exactly as written. So a typo in a directive produces braces in the output and **no error +at all**. The slides half refuses the same name and names the alternative. See +`references/directive-rules.md`. + +## The command line + +```bash +mdoffice word to-docx notes.md # to .docx +mdoffice word to-odt notes.md # to .odt +mdoffice word to-rtf notes.md # to .rtf +mdoffice word to-markdown notes.docx # and back +``` + +It decides which way the data has to go **from the content, not the name**: a Word document +is a zip and Markdown is text, and the four bytes that say which is which are part of the +format. Exit `0` on success and `1` for anything that did not finish — a mistake the caller +can put right or a defect they cannot. + +The command line is not part of this library; it is `mdoffice`, one program over both +converters. `references/command-line-install.md` says how to get it. diff --git a/skill/mdword/pages.json b/skill/mdword/pages.json new file mode 100644 index 0000000..afbff8f --- /dev/null +++ b/skill/mdword/pages.json @@ -0,0 +1,85 @@ +{ + "_comment": [ + "The pages the bundle carries, and the files they are carried in.", + "", + "The bundle is a copy of this site's documentation, made by the build. This file is", + "what says which pages are in it \u2014 and it is read by the build that writes the bundle", + "and by the checker that refuses a bundle that has fallen behind.", + "", + "The 'sections' are the pages that must all be there. A page added to one of them and", + "not carried is a page the bundle has silently fallen behind on, and the checker says", + "so rather than letting it pass. The pages outside them \u2014 the command line for the", + "slides half, and the suite itself \u2014 are carried or not as this file says, because a", + "skill about the Word half has no reason to carry the slides half's command line.", + "", + "The 'file' is the name the page is carried under. It is not the page's URL: the pages", + "are flattened into one directory, and two pages of different sections can want the", + "same name, so the names say which section a page is from where that is not obvious." + ], + "sections": [ + "Markdown Word", + "In general" + ], + "references": [ + { + "file": "dialect.md", + "page": "word/index.md" + }, + { + "file": "directives.md", + "page": "word/directives.md" + }, + { + "file": "configuration.md", + "page": "word/configuration.md" + }, + { + "file": "templates.md", + "page": "word/templates.md" + }, + { + "file": "round-trip.md", + "page": "word/round-trip.md" + }, + { + "file": "images.md", + "page": "word/images.md" + }, + { + "file": "errors.md", + "page": "word/errors.md" + }, + { + "file": "directive-rules.md", + "page": "general/directives.md" + }, + { + "file": "directive-syntax.md", + "page": "general/syntax.md" + }, + { + "file": "general-images.md", + "page": "general/images.md" + }, + { + "file": "errors-general.md", + "page": "general/errors.md" + }, + { + "file": "command-line-word.md", + "page": "command-line/word.md" + }, + { + "file": "command-line-basic.md", + "page": "command-line/basic.md" + }, + { + "file": "command-line-install.md", + "page": "command-line/installation.md" + }, + { + "file": "install.md", + "page": "skills/index.md" + } + ] +} diff --git a/skill/mdword/scripts/render.php b/skill/mdword/scripts/render.php new file mode 100644 index 0000000..dda313a --- /dev/null +++ b/skill/mdword/scripts/render.php @@ -0,0 +1,363 @@ +#!/usr/bin/env php + --port [--out ] + * + * **A document that has not been looked at has not been checked.** This is the tool that + * looks at one: it opens the document in an application that lays it out the way Word does + * and writes a PNG of every page, so that the pages can be read. + * + * The application is driven through its control protocol: one JSON object per line over a + * loopback port, each request carrying the key the application wrote when it bound the + * port. The protocol is documented upstream, at + * `github.com/storytold/wordcraft/docs/control-protocol.md` and + * `github.com/storytold/deckcraft/docs/control-protocol.md`. + * + * **The two applications do not speak the same protocol.** They speak the same *shape* — + * one JSON object per line, a key with every request, `{"id": …, "ok": true, "result": …}` + * or `{"ok": false, "error": …}` in reply — and they differ in their method names: + * WordCraft opens a document with `file.open` and renders a page with `ui.render` and a + * `page`, while DeckCraft opens one with `app.open` and renders a slide with `ui.render` + * and a `slide`. So this tries one and falls back to the other, and says which it settled + * on. + * + * The port is the application's own: start it with `--control ` and read the key out + * of the file it writes. Nothing here starts an application, because an application that + * answers a question is a question somebody asked on purpose. + */ + +const HOST = '127.0.0.1'; + +/** + * A render that did not happen, and why. + * + * A dedicated class rather than the generic one, because the message is the whole of what a + * caller gets: a script that says "something went wrong" has said nothing. + */ +final class RenderFailed extends RuntimeException +{ +} + +/** + * The settings folder each application keeps its control key in. + * + * The file is named after the port it belongs to, so two applications on two ports do not + * share one — and one application restarted on the same port writes a new key, so the key + * is read at the moment it is used rather than once at the start. + * + * `AppData/Roaming` is where a Windows application keeps what it would keep in + * `Library/Application Support` on a Mac, which is the same thing under another name. + */ +const KEY_FILES = [ + 'word' => [ + 'Darwin' => 'Library/Application Support/WordCraft/control-key.local-%d', + 'Linux' => '.config/wordcraft/control-key.local-%d', + 'Windows' => 'AppData/Roaming/WordCraft/control-key.local-%d', + ], + 'deck' => [ + 'Darwin' => 'Library/Application Support/DeckCraft/control-key.local-%d', + 'Linux' => '.config/deckcraft/control-key.local-%d', + 'Windows' => 'AppData/Roaming/DeckCraft/control-key.local-%d', + ], +]; + +/** + * The control key for a port. + * + * Which application the key belongs to is not decided here: the key file says where to + * look for the key, and the method that answers the socket says what is on the other end + * of it. + */ +function controlKey(int $port): string +{ + $home = (string) (\getenv('HOME') ?: \getenv('USERPROFILE') ?: ''); + + if ($home === '') { + throw new RenderFailed('The home folder could not be found, so the control key could not be.'); + } + + $system = \PHP_OS_FAMILY; + + foreach (KEY_FILES as $patterns) { + $pattern = $patterns[$system] ?? null; + + if ($pattern === null) { + continue; + } + + $path = $home . '/' . \sprintf($pattern, $port); + + if (! \is_file($path)) { + continue; + } + + $key = \trim((string) \file_get_contents($path)); + + if ($key === '') { + continue; + } + + return $key; + } + + throw new RenderFailed(\sprintf( + 'No control key was found for port %d. Start the application with `--control %d` and try again.', + $port, + $port, + )); +} + +/** + * A connection to an application, and the key it wants. + */ +function connect(int $port): mixed +{ + $socket = @\stream_socket_client(\sprintf('tcp://%s:%d', HOST, $port), $errno, $error, 10); + + if ($socket === false) { + throw new RenderFailed(\sprintf( + 'Nothing is listening on port %d (%s). Start the application with `--control %d` first.', + $port, + $error, + $port, + )); + } + + return $socket; +} + +/** + * One request, and the reply as the application sent it. + * + * **The reply is checked before it is read.** The protocol says a reply carries the id of + * the request it answers, and a client is to check it: a reply to an earlier request is + * not the answer to this one. Every request therefore carries an id of its own, and + * every reply is checked against the id that was sent. + * + * @param array $params + * + * @return array + */ +function send(mixed $socket, string $key, string $method, array $params = []): array +{ + static $requests = 0; + + $id = ++$requests; + + $request = ['id' => $id, 'key' => $key, 'method' => $method]; + + if ($params !== []) { + $request['params'] = $params; + } + + \fwrite($socket, \json_encode($request) . "\n"); + + $line = \fgets($socket); + + if ($line === false) { + throw new RenderFailed(\sprintf('The application closed the connection during %s.', $method)); + } + + $reply = \json_decode($line, true); + + if (! \is_array($reply)) { + throw new RenderFailed(\sprintf('The reply to %s was not JSON.', $method)); + } + + if (($reply['id'] ?? null) !== $id) { + throw new RenderFailed(\sprintf( + 'The application answered %s while %d was asked, so the reply is not the answer to the question.', + \json_encode($reply['id'] ?? null), + $id, + )); + } + + return $reply; +} + +/** + * One request, and the result it answers with. + * + * @param array $params + * + * @return array + */ +function ask(mixed $socket, string $key, string $method, array $params = []): array +{ + $reply = send($socket, $key, $method, $params); + + if (($reply['ok'] ?? false) !== true) { + throw new RenderFailed(\sprintf( + '%s: %s', + $method, + (string) ($reply['error'] ?? 'the application said nothing about why'), + )); + } + + return (array) ($reply['result'] ?? []); +} + +/** + * Opens a document, in whichever of the two applications answers. + * + * **The method is found by asking, and not by probing.** WordCraft opens a document with + * `file.open` and DeckCraft with `app.open`, and the two know nothing of each other. So + * `file.open` is asked for with the document, and `app.open` is asked for only when the + * first answer says the method is unknown — which is the one failure that means "ask the + * other one" rather than "something is wrong". + * + * The application that answered is the one that renders: the key file says where to look + * for the key, and the method that answers says what is on the other end of the socket. + * + * @return array{app: string, opened: string} + */ +function open(mixed $socket, string $key, string $document): array +{ + foreach (['file.open' => 'word', 'app.open' => 'deck'] as $method => $app) { + $reply = send($socket, $key, $method, ['path' => $document]); + + if (($reply['ok'] ?? false) === true) { + $result = (array) ($reply['result'] ?? []); + + return ['app' => $app, 'opened' => (string) ($result['path'] ?? $document)]; + } + + $error = (string) ($reply['error'] ?? ''); + + if (\stripos($error, 'unknown command') === false) { + throw new RenderFailed(\sprintf( + '%s: %s', + $method, + $error === '' ? 'the application said nothing about why' : $error, + )); + } + } + + throw new RenderFailed('The application answered, but it knows neither `file.open` nor `app.open`.'); +} + +/** + * How many pages a document has. + * + * @return int + */ +function pages(mixed $socket, string $key): int +{ + $reply = ask($socket, $key, 'document.inspect'); + + foreach (['pages', 'slides'] as $field) { + if (isset($reply[$field]) && \is_numeric($reply[$field])) { + return (int) $reply[$field]; + } + } + + // A document of one page is the answer when the application will not say, and it is + // the answer that is right more often than not. + return 1; +} + +/** + * Renders a document to a picture of every page. + * + * @param list $written + */ +function render(mixed $socket, string $key, string $document, string $directory): array +{ + if (! \is_dir($directory) && ! @\mkdir($directory, 0o777, true) && ! \is_dir($directory)) { + throw new RenderFailed(\sprintf('The directory "%s" could not be made.', $directory)); + } + + $opened = open($socket, $key, $document); + $app = $opened['app']; + + $count = pages($socket, $key); + $written = []; + $stem = \pathinfo($document, PATHINFO_FILENAME); + + for ($page = 1; $page <= $count; $page++) { + $path = \sprintf('%s/%s-page-%02d.png', \rtrim($directory, '/'), $stem, $page); + + // WordCraft counts pages from one and calls them pages; DeckCraft counts slides + // from one and calls them slides. The parameter is the only difference. + $params = $app === 'deck' + ? ['path' => $path, 'slide' => $page] + : ['path' => $path, 'page' => $page]; + + ask($socket, $key, 'ui.render', $params); + + if (! \is_file($path)) { + throw new RenderFailed(\sprintf('The application wrote no picture for page %d.', $page)); + } + + $written[] = $path; + } + + return $written; +} + +// +// The run. +// + +$arguments = \array_slice($argv ?? [], 1); +$document = null; +$directory = \sys_get_temp_dir(); +$port = null; + +// The arguments are read with a cursor rather than by moving the loop's own counter: a +// counter that moves twice is a counter that is wrong once. +$cursor = 0; + +while ($cursor < \count($arguments)) { + $argument = $arguments[$cursor]; + + if ($argument === '--out') { + $directory = $arguments[$cursor + 1] ?? $directory; + $cursor += 2; + + continue; + } + + if ($argument === '--port') { + $port = (int) ($arguments[$cursor + 1] ?? 0); + $cursor += 2; + + continue; + } + + $document ??= $argument; + $cursor++; +} + +if ($document === null) { + \fwrite(STDERR, "Usage: php render.php --port [--out ]\n"); + exit(1); +} + +if ($port === null || $port === 0) { + \fwrite(STDERR, "The port is missing. Start the application with `--control ` and pass it.\n"); + exit(1); +} + +try { + $key = controlKey($port); + $socket = connect($port); + + try { + $written = render($socket, $key, $document, $directory); + } finally { + \fclose($socket); + } +} catch (Throwable $problem) { + \fwrite(STDERR, $problem->getMessage() . "\n"); + exit(1); +} + +foreach ($written as $path) { + \printf("%s\n", $path); +} diff --git a/tests/Claims/word-templates.php b/tests/Claims/word-templates.php new file mode 100644 index 0000000..7139d93 --- /dev/null +++ b/tests/Claims/word-templates.php @@ -0,0 +1,108 @@ +toBe('body'); +}); + +it('matches a control on its tag rather than on its alias', function () { + says(Site::WORD_TEMPLATES, 'The control is matched on its **tag**'); + + // The control's alias says something else on purpose: a library that matched on the + // alias would not find this one, and the content would go to the end of the body + // instead of into the control. + $template = Template::withControl('body', 'A different label'); + + $output = Template::written((new MarkdownTemplate($template))->insert(CONTENT)); + + expect(Template::xmlOf($output))->toContain(CONTENT) + ->and(Template::xmlOf($output))->not->toContain('w:alias w:val="A different label"'); +}); + +it('says that filled fields are unlinked, and takes the field away', function () { + says(Site::WORD_TEMPLATES, 'Filled fields are unlinked'); + + $template = Template::withControl(); + + $output = Template::written( + (new MarkdownTemplate($template)) + ->fields(['reference' => 'REF-1']) + ->insert(CONTENT), + ); + + $xml = Template::xmlOf($output); + + expect($xml)->toContain('REF-1') + ->and($xml)->not->toContain('MERGEFIELD'); +}); + +it('says that a template with no control takes the content at the end of its body', function () { + says(Site::WORD_TEMPLATES, 'takes the content **at the end of its body**'); + + $template = Template::withoutControl(); + + $output = Template::written((new MarkdownTemplate($template))->insert(CONTENT)); + + expect(Template::textOf($output))->toContain(CONTENT); +}); + +it('says that a template with no control can be refused instead, and refuses it by name', function () { + says(Site::WORD_TEMPLATES, 'says so and refuses'); + + $template = Template::withoutControl(); + + // The refusal comes when the document is written rather than when the content is + // put in: the content has somewhere to go until the moment it has to be there. + expect( + fn () => (new MarkdownTemplate($template)) + ->contentFallback(ContentFallback::Error) + ->insert(CONTENT) + ->toString(), + )->toThrow(NoContentControl::class); +}); + +it('says that a filled-in template is always a document, and writes one', function () { + says(Site::WORD_TEMPLATES, 'a filled-in template is always a document'); + + $template = Template::dotx(Template::withControl()); + + $output = Template::written((new MarkdownTemplate($template))->insert(CONTENT)); + + // The result is a document whatever went in, and the file says so: the content type + // is the one a document has and not the one a template has. + $zip = new \ZipArchive(); + $zip->open($output); + $declared = (string) $zip->getFromName('[Content_Types].xml'); + $zip->close(); + + expect($declared)->toContain('wordprocessingml.document.main+xml') + ->and($declared)->not->toContain('wordprocessingml.template.main+xml'); +}); diff --git a/tests/Skill/render.php b/tests/Skill/render.php new file mode 100644 index 0000000..83a441f --- /dev/null +++ b/tests/Skill/render.php @@ -0,0 +1,187 @@ + $environment + * @param list $arguments + * + * @return array{out: string, err: string, status: int} + */ +function client(array $environment, array $arguments): array +{ + $root = dirname(__DIR__, 2); + + $descriptors = [ + 1 => ['pipe', 'w'], + 2 => ['pipe', 'w'], + ]; + + $process = proc_open( + [PHP_BINARY, $root . '/skill/mdword/scripts/render.php', ...$arguments], + $descriptors, + $pipes, + null, + array_merge(getenv(), $environment), + ); + + if (! is_resource($process)) { + throw TheMachineryFailed::because('The client could not be started.'); + } + + $out = (string) stream_get_contents($pipes[1]); + $err = (string) stream_get_contents($pipes[2]); + + fclose($pipes[1]); + fclose($pipes[2]); + + return ['out' => $out, 'err' => $err, 'status' => proc_close($process)]; +} + +/** + * The control key the client reads, where the application that wrote it keeps it. + * + * The client looks in the same place, and a key that is somewhere else is a client that + * cannot find it — which is the one thing this test would be measuring rather than the + * thing it is for. + */ +function writeControlKey(string $home, int $port): void +{ + $relative = match (PHP_OS_FAMILY) { + 'Darwin' => 'Library/Application Support/WordCraft', + 'Linux' => '.config/wordcraft', + 'Windows' => 'AppData/Roaming/WordCraft', + default => null, + }; + + if ($relative === null) { + test()->markTestSkipped('This platform keeps the control key somewhere this does not know.'); + } + + $directory = $home . '/' . $relative; + + if (! is_dir($directory) && ! mkdir($directory, 0o777, true) && ! is_dir($directory)) { + throw TheMachineryFailed::because(sprintf('The directory "%s" could not be made.', $directory)); + } + + file_put_contents($directory . '/control-key.local-' . $port, 'the-key'); +} + +/** + * A home folder, a document and a directory to write to, for one run. + * + * @return array{home: string, document: string, out: string} + */ +function aRun(): array +{ + $unique = bin2hex(random_bytes(6)); + + $home = sys_get_temp_dir() . '/mdword-home-' . $unique; + $out = sys_get_temp_dir() . '/mdword-out-' . $unique; + + mkdir($home, 0o777, true); + mkdir($out, 0o777, true); + + $document = $home . '/notes.docx'; + + file_put_contents($document, 'not a document'); + + return ['home' => $home, 'document' => $document, 'out' => $out]; +} + +it('opens the document with one request that carries the path', function () { + $server = ControlServer::speaking('word'); + $run = aRun(); + + writeControlKey($run['home'], $server->port); + + $answer = client( + ['HOME' => $run['home']], + [$run['document'], PORT_FLAG, (string) $server->port, OUT_FLAG, $run['out']], + ); + + expect($answer['status'])->toBe(0) + ->and($server->requests()[0])->toBe('file.open {"path":"' . $run['document'] . '"}') + ->and(glob($run['out'] . '/*.png'))->not->toBeEmpty(); +}); + +it('opens the document with the other method when the first is unknown', function () { + // DeckCraft opens a document with `app.open` and knows nothing of `file.open`, so + // the client asks the other one only when the first is refused. + $server = ControlServer::speaking('deck'); + $run = aRun(); + + writeControlKey($run['home'], $server->port); + + $answer = client( + ['HOME' => $run['home']], + [$run['document'], PORT_FLAG, (string) $server->port, OUT_FLAG, $run['out']], + ); + + expect($answer['status'])->toBe(0) + ->and($server->requests()[0])->toBe('file.open {"path":"' . $run['document'] . '"}') + ->and($server->requests()[1])->toBe('app.open {"path":"' . $run['document'] . '"}'); +}); + +it('refuses a reply that answers another question', function () { + $server = ControlServer::speaking('bad-id'); + $run = aRun(); + + writeControlKey($run['home'], $server->port); + + $answer = client( + ['HOME' => $run['home']], + [$run['document'], PORT_FLAG, (string) $server->port, OUT_FLAG, $run['out']], + ); + + expect($answer['status'])->not->toBe(0) + ->and($answer['err'])->toContain('not the answer'); +}); + +it('says so when there is no application to talk to', function () { + $run = aRun(); + + // A port nothing is listening on. + $answer = client( + ['HOME' => $run['home']], + [$run['document'], PORT_FLAG, '1', OUT_FLAG, $run['out']], + ); + + expect($answer['status'])->not->toBe(0) + ->and($answer['err'])->not->toBe(''); +}); diff --git a/tests/Support/ControlServer.php b/tests/Support/ControlServer.php new file mode 100644 index 0000000..dd19779 --- /dev/null +++ b/tests/Support/ControlServer.php @@ -0,0 +1,247 @@ + $mode !== 'deck', + 'app.open' => $mode === 'deck', + 'document.inspect', 'ui.render' => true, + default => false, + }; + + if (! $known) { + fwrite($client, json_encode([ + 'id' => $id, + 'ok' => false, + 'error' => 'unknown command: ' . $method, + ]) . "\n"); + + continue; + } + + $result = []; + + if ($method === 'file.open' || $method === 'app.open') { + $result = ['path' => $request['params']['path'] ?? '']; + } + + if ($method === 'document.inspect') { + $result = ['pages' => 1]; + } + + if ($method === 'ui.render') { + file_put_contents((string) ($request['params']['path'] ?? ''), $picture); + } + + fwrite($client, json_encode(['id' => $id, 'ok' => true, 'result' => $result]) . "\n"); + } + + fclose($client); + } + PHP; + + /** + * The port the server answers on. + */ + public int $port = 0; + + /** + * Where every request it was sent is written down. + */ + public string $log; + + /** + * @var resource|null + */ + private $process = null; + + /** + * @var array + */ + private array $pipes = []; + + private function __construct() + { + $log = \tempnam(\sys_get_temp_dir(), 'mdword-control-'); + + if ($log === false) { + throw TheMachineryFailed::because('The control server could not be given a file to write to.'); + } + + $this->log = $log; + } + + /** + * A server on a port of its own, answering the protocol. + */ + public static function speaking(string $mode): self + { + $server = new self(); + + $portFile = \tempnam(\sys_get_temp_dir(), 'mdword-port-'); + + if ($portFile === false) { + throw TheMachineryFailed::because('The control server could not be given a file to write its port to.'); + } + + $server->process = \proc_open( + [\PHP_BINARY, '-r', self::CODE], + [ + 1 => ['pipe', 'w'], + 2 => ['pipe', 'w'], + ], + $server->pipes, + null, + \array_merge(\getenv(), [ + 'MDWORD_MODE' => $mode, + 'MDWORD_PORT_FILE' => $portFile, + 'MDWORD_LOG_FILE' => $server->log, + ]), + ); + + if (! \is_resource($server->process)) { + throw TheMachineryFailed::because('The control server could not be started.'); + } + + $server->port = $server->waitForPort($portFile); + + return $server; + } + + /** + * Every request the server was sent, in the order they arrived. + * + * @return list + */ + public function requests(): array + { + $written = \file_get_contents($this->log); + + if ($written === false) { + return []; + } + + return \array_values(\array_filter(\explode("\n", $written))); + } + + /** + * Stops the server. + */ + public function stop(): void + { + if ($this->process === null) { + return; + } + + foreach ($this->pipes as $pipe) { + if (\is_resource($pipe)) { + \fclose($pipe); + } + } + + \proc_terminate($this->process); + \proc_close($this->process); + + $this->process = null; + } + + public function __destruct() + { + $this->stop(); + } + + /** + * The port the server bound, which it writes down because it chose it itself. + * + * The server binds to port zero and takes what it is given, rather than being told + * a port to bind: a port chosen here and handed over is a port another process may + * have taken in between. + */ + private function waitForPort(string $portFile): int + { + for ($tries = 0; $tries < 100; $tries++) { + $written = \trim((string) \file_get_contents($portFile)); + + if ($written !== '') { + return (int) $written; + } + + \usleep(50000); + } + + throw TheMachineryFailed::because('The control server did not say which port it is on.'); + } +} diff --git a/tests/Support/Site.php b/tests/Support/Site.php index 72c3849..653eab5 100644 --- a/tests/Support/Site.php +++ b/tests/Support/Site.php @@ -39,6 +39,11 @@ final class Site */ public const WORD_ROUND_TRIP = 'word/round-trip.md'; + /** + * The page that documents rendering into a design that already exists. + */ + public const WORD_TEMPLATES = 'word/templates.md'; + /** * The root of the site: the directory this repository is in. */ diff --git a/tests/Support/Template.php b/tests/Support/Template.php new file mode 100644 index 0000000..66fcb03 --- /dev/null +++ b/tests/Support/Template.php @@ -0,0 +1,281 @@ +open($path); + + $declared = (string) $zip->getFromName(self::TYPES_PART); + + $zip->deleteName(self::TYPES_PART); + $zip->addFromString( + self::TYPES_PART, + \str_replace( + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml', + 'application/vnd.openxmlformats-officedocument.wordprocessingml.template.main+xml', + $declared, + ), + ); + + $zip->close(); + + return $path; + } + + /** + * The document a template made, written out. + */ + public static function written(MarkdownTemplate $template): string + { + $path = \sys_get_temp_dir() . '/markdown-office-written-' . \bin2hex(\random_bytes(6)) . self::EXTENSION; + + $template->save($path); + + return $path; + } + + /** + * The text a document holds, as the file holds it. + */ + public static function textOf(string $path): string + { + return Package::part($path, self::BODY_PART); + } + + /** + * The body of a document, as the XML it was written as. + */ + public static function xmlOf(string $path): string + { + return Package::part($path, self::BODY_PART); + } + + /** + * A `.docx` with a paragraph to put things into. + */ + private static function base(): string + { + $phpWord = new PhpWord(); + $section = $phpWord->addSection(); + + $section->addText('Northwind Ltd'); + $section->addText('@@reference@@'); + $section->addText('PLACEHOLDER'); + + $path = \sys_get_temp_dir() . self::STEM . \bin2hex(\random_bytes(6)) . self::EXTENSION; + + $writer = \PhpOffice\PhpWord\IOFactory::createWriter($phpWord, 'Word2007'); + $writer->save($path); + + return $path; + } + + /** + * The document with the field written into it. + */ + private static function withFieldMarkup(string $path): string + { + return self::rewrite($path, static function (\DOMDocument $dom): void { + $paragraph = self::paragraphWith($dom, '@@reference@@'); + + if ($paragraph === null) { + return; + } + + $field = $dom->createElementNS(self::W_NS, 'w:fldSimple'); + $field->setAttributeNS(self::W_NS, 'w:instr', ' MERGEFIELD reference \\* MERGEFORMAT '); + + $run = $dom->createElementNS(self::W_NS, 'w:r'); + $text = $dom->createElementNS(self::W_NS, 'w:t'); + $text->appendChild($dom->createTextNode('«reference»')); + + $run->appendChild($text); + $field->appendChild($run); + + $paragraph->parentNode?->replaceChild($field, $paragraph); + }); + } + + /** + * The document with the control written into it. + */ + private static function withControlMarkup(string $path, string $tag, string $alias): string + { + $markup = ContentControl::markup($tag); + + if ($alias !== '') { + $markup = \str_replace( + '', + '', + $markup, + ); + } + + return self::rewrite($path, static function (\DOMDocument $dom) use ($markup): void { + $paragraph = self::paragraphWith($dom, 'PLACEHOLDER'); + + if ($paragraph === null) { + return; + } + + $fragment = new \DOMDocument(); + $fragment->loadXML( + '' . $markup . '', + ); + + $control = $fragment->documentElement?->firstChild; + + if ($control === null) { + return; + } + + $imported = $dom->importNode($control, true); + $paragraph->parentNode?->replaceChild($imported, $paragraph); + }); + } + + /** + * The paragraph of a document that holds the given text. + */ + private static function paragraphWith(\DOMDocument $dom, string $text): ?\DOMElement + { + $xpath = new \DOMXPath($dom); + $xpath->registerNamespace('w', self::W_NS); + + foreach ($xpath->query('//w:p') ?: [] as $paragraph) { + if ($paragraph->textContent === $text) { + \assert($paragraph instanceof \DOMElement); + + return $paragraph; + } + } + + return null; + } + + /** + * The document, rewritten part by part. + */ + private static function rewrite(string $path, callable $change): string + { + $zip = new \ZipArchive(); + $zip->open($path); + + $xml = (string) $zip->getFromName(self::BODY_PART); + $zip->close(); + + $dom = new \DOMDocument(); + $dom->loadXML($xml); + + $change($dom); + + $written = \sys_get_temp_dir() . self::STEM . \bin2hex(\random_bytes(6)) . self::EXTENSION; + + \copy($path, $written); + + $out = new \ZipArchive(); + $out->open($written); + $out->deleteName(self::BODY_PART); + $out->addFromString(self::BODY_PART, (string) $dom->saveXML()); + $out->close(); + + return $written; + } + + /** + * A copy of a document under a name of its own. + */ + private static function copied(string $path, string $name): string + { + $written = \sys_get_temp_dir() . '/markdown-office-' . $name . '-' . \bin2hex(\random_bytes(6)) . self::EXTENSION; + + \copy($path, $written); + + return $written; + } +} diff --git a/tests/Support/TheMachineryFailed.php b/tests/Support/TheMachineryFailed.php new file mode 100644 index 0000000..ae984d6 --- /dev/null +++ b/tests/Support/TheMachineryFailed.php @@ -0,0 +1,20 @@ +, references: list} + */ +function manifest(string $root): array +{ + $file = $root . '/skill/mdword/pages.json'; + + if (!\is_file($file)) { + throw new UnreadableSkill('skill/mdword/pages.json could not be read.'); + } + + $parsed = \json_decode((string) \file_get_contents($file), true); + + if (!\is_array($parsed)) { + throw new UnreadableSkill('skill/mdword/pages.json could not be read as JSON.'); + } + + return [ + 'sections' => (array) ($parsed['sections'] ?? []), + 'references' => (array) ($parsed['references'] ?? []), + ]; +} + +/** + * Every page of the site, with its section. + * + * @return array + */ +function pages(string $content): array +{ + $found = []; + + foreach (\glob($content . '/*/*.md') ?: [] as $file) { + $source = (string) \file_get_contents($file); + + if (\preg_match('/^section:\s*(.+)$/m', $source, $section) !== 1) { + continue; + } + + $found[\str_replace($content . '/', '', $file)] = \trim($section[1]); + } + + return $found; +} + +/** + * Every file in a directory, recursively. + * + * @return list + */ +function files(string $directory): array +{ + $found = []; + + foreach (\glob($directory . '/*') ?: [] as $path) { + if (\is_dir($path)) { + $found = \array_merge($found, files($path)); + + continue; + } + + $found[] = $path; + } + + return $found; +} + +/** + * The address a page is served at, as its own frontmatter declares it. + * + * The mirror of a page is written beside its HTML, so it is at the page's permalink and + * not at the page's source path — which are not the same thing: a page in `general/` may + * be served at `/directives/`. + */ +function permalink(string $page): string +{ + $source = (string) \file_get_contents($page); + + if (\preg_match('/^permalink:\s*(\S+)\s*$/m', $source, $found) !== 1) { + throw new UnreadableSkill(\sprintf('The page "%s" has no permalink.', $page)); + } + + return $found[1]; +} + +/** + * The address the site is served from. + * + * Read out of the configuration rather than written down, because the configuration is what + * Eleventy reads and a check that disagrees with it is a check of a different site. + */ +function canonical(string $root): string +{ + $given = \getenv('MARKDOWN_OFFICE_CANONICAL'); + + if (\is_string($given) && $given !== '') { + return \rtrim($given, '/'); + } + + $source = (string) \file_get_contents($root . '/eleventy.config.js'); + + if (\preg_match('/MARKDOWN_OFFICE_CANONICAL\s*\?\?\s*"([^"]+)"/', $source, $found) !== 1) { + throw new UnreadableSkill('The configuration does not say what the canonical host is.'); + } + + return \rtrim($found[1], '/'); +} + +/** + * The body of a page, with its frontmatter off. + */ +function body(string $source): string +{ + $parts = \preg_match('/^---\r?\n[\s\S]*?\r?\n---\r?\n?([\s\S]*)$/', $source, $found); + + return $parts === 1 ? $found[1] : $source; +} + +/** + * The example blocks of a page, and the prose between them. + * + * @return array{examples: list, prose: list} + */ +function blocks(string $source): array +{ + $parts = \preg_split( + '/\{%\s*example[^%]*%\}\n?([\s\S]*?)\n?\{%\s*endexample\s*%\}/', + body($source), + -1, + \PREG_SPLIT_DELIM_CAPTURE, + ) ?: []; + + $examples = []; + $prose = []; + + foreach ($parts as $index => $part) { + // The odd parts are the fences and the even parts are the prose around them. + if ($index % 2 === 1) { + $examples[] = \trim($part); + continue; + } + + $prose[] = \trim($part); + } + + return ['examples' => $examples, 'prose' => $prose]; +} + +/** + * The paths a reference file names. + * + * @return list + */ +function paths(string $reference): array +{ + \preg_match_all('/`((?:examples|references|scripts)\/[^`]+)`/', $reference, $found); + + return $found[1]; +} + +$manifest = manifest($root); +$pages = pages($root . '/content'); + +// 1. Every reference file is the page it carries. +foreach ($manifest['references'] as $reference) { + $file = $bundle . '/references/' . $reference['file']; + + if (!\is_file($file)) { + $report->fail(\sprintf('The bundle does not carry "%s".', $reference['file'])); + + continue; + } + + $mirror = $site . '/' . \ltrim(permalink($root . '/content/' . $reference['page']), '/') . 'index.md'; + + if (!\is_file($mirror)) { + $report->fail(\sprintf('The page "%s" has no Markdown mirror to copy.', $reference['page'])); + + continue; + } + + $written = (string) \file_get_contents($file); + $source = (string) \file_get_contents($mirror); + $blocks = blocks($source); + + foreach ($blocks['prose'] as $piece) { + if ($piece === '') { + continue; + } + + if (!\str_contains($written, $piece)) { + $report->fail(\sprintf( + 'The reference file "%s" does not carry the text of "%s". It is a copy of the page, ' + . 'not a paraphrase of it.', + $reference['file'], + $reference['page'], + )); + } + } + + $expansions = \preg_match_all('/^#### Example: /m', $written); + + if ($expansions !== \count($blocks['examples'])) { + $report->fail(\sprintf( + 'The reference file "%s" expands %d of the %d example(s) of "%s".', + $reference['file'], + $expansions, + \count($blocks['examples']), + $reference['page'], + )); + } +} + +// 2. Every page of the sections the bundle claims to cover is carried. +$carried = \array_column($manifest['references'], 'page'); + +foreach ($pages as $page => $section) { + if (!\in_array($section, $manifest['sections'], true)) { + continue; + } + + if (!\in_array($page, $carried, true)) { + $report->fail(\sprintf( + 'The page "%s" is in the "%s" section and the bundle does not carry it.', + $page, + $section, + )); + } +} + +// 3. The catalogue and the bundle agree. +$catalogue = \json_decode((string) \file_get_contents($site . '/skills/index.json'), true); + +if (!\is_array($catalogue) || !\is_array($catalogue['skills'] ?? null)) { + $report->fail('The catalogue at /skills/index.json is not a catalogue.'); +} else { + $entry = $catalogue['skills'][0] ?? []; + + if (($entry['name'] ?? null) !== 'mdword') { + $report->fail('The catalogue does not carry a skill called "mdword".'); + } + + $named = (array) ($entry['files'] ?? []); + $there = []; + + foreach (files($bundle) as $file) { + $there[] = \str_replace($bundle . '/', '', $file); + } + + \sort($there); + + foreach ($there as $file) { + if (!\in_array($file, $named, true)) { + $report->fail(\sprintf('The bundle holds "%s" and the catalogue does not name it.', $file)); + } + } + + foreach ($named as $file) { + if (!\in_array($file, $there, true)) { + $report->fail(\sprintf('The catalogue names "%s" and the bundle does not hold it.', $file)); + } + } +} + +// 4. Every path a reference file names exists. +foreach (\glob($bundle . '/references/*.md') ?: [] as $file) { + foreach (paths((string) \file_get_contents($file)) as $path) { + if (!\file_exists($bundle . '/' . $path)) { + $report->fail(\sprintf( + 'The reference file "%s" names "%s" and the bundle does not hold it.', + \basename($file), + $path, + )); + } + } +} + +// 5. The skill's frontmatter. +$skill = (string) \file_get_contents($bundle . '/SKILL.md'); + +if (\preg_match('/^---\r?\n(.*?)\r?\n---/s', $skill, $frontmatter) !== 1) { + $report->fail('The skill has no frontmatter.'); +} else { + $head = $frontmatter[1]; + + if (\preg_match('/^name:\s*(\S+)$/m', $head, $name) !== 1) { + $report->fail('The skill has no name.'); + } + + if (\preg_match('/^description:\s*(.+)$/m', $head, $description) !== 1) { + $report->fail('The skill has no description, so it is never advertised to the model.'); + } elseif (!\str_contains($description[1], 'Use when')) { + $report->fail('The skill does not say when to use it.'); + } + + // Every reference the bundle carries is named in the skill, because a file nothing + // points at is a file nobody reads. + foreach (\glob($bundle . '/references/*.md') ?: [] as $file) { + $name = \basename($file); + + if (!\str_contains($skill, 'references/' . $name)) { + $report->fail(\sprintf('The skill does not point at "references/%s".', $name)); + } + } +} + +// 6. The scripts are the ones the library ships. +$library = $root . '/../markdown-word/tools'; + +/** + * The tools the bundle carries from the library. + * + * Named once, so that the list that is checked and the count that is reported cannot + * disagree about what is in it. + * + * @var list + */ +$scripts = [ + 'render-document.php', + 'render-with-word.applescript', + 'render-with-word.ps1', + 'package-as-document.sh', + 'pdf-is-complete.sh', +]; + +foreach ($scripts as $tool) { + $from = $library . '/' . $tool; + $copied = $bundle . '/scripts/' . $tool; + + if (!\is_file($from)) { + $report->fail(\sprintf('The library does not ship "%s".', $tool)); + + continue; + } + + if (!\is_file($copied)) { + $report->fail(\sprintf('The bundle does not carry the script "%s".', $tool)); + + continue; + } + + if (\md5_file($from) !== \md5_file($copied)) { + $report->fail(\sprintf('The bundle\'s copy of "%s" is not the one the library ships.', $tool)); + } +} + +// 7. Every path the install page names exists in the built tree. +$install = (string) \file_get_contents($root . '/' . SKILL_PAGE); + +\preg_match_all('#https?://[^/\s]+/skills/([^\s`)\]",]+)#', $install, $found); + +foreach ($found[1] as $path) { + if (\str_contains($path, '/')) { + // A path under the bundle, which is a directory rather than a file. + if (!\file_exists($site . '/skills/' . $path)) { + $report->fail(\sprintf('The install page names "/skills/%s" and it is not there.', $path)); + } + + continue; + } + + if (!\file_exists($site . '/skills/' . $path)) { + $report->fail(\sprintf('The install page names "/skills/%s" and it is not there.', $path)); + } +} + +// 8. The library's README points at the skill page. +// +// The address is read out of the site's own page rather than written down, so a page that +// moves does not leave this pointing at a page that is no longer there — the pattern +// `check-readme-links.php` already uses for the command-line page. +$readme = $root . '/../markdown-word/README.md'; + +if (\is_file($readme)) { + $text = (string) \file_get_contents($readme); + + if (!\str_contains($text, canonical($root) . permalink($root . '/' . SKILL_PAGE))) { + $report->fail(\sprintf( + 'The library\'s README does not point at the skill page, which is at %s.', + canonical($root) . permalink($root . '/' . SKILL_PAGE), + )); + } +} + +if ($report->failed()) { + $report->print('skill'); + + exit(1); +} + +\printf( + "%d reference file(s), %d page(s) and %d script(s) checked against the bundle.\n", + \count($manifest['references']), + \count($pages), + \count($scripts), +); diff --git a/tools/lib/UnreadableSkill.php b/tools/lib/UnreadableSkill.php new file mode 100644 index 0000000..2808392 --- /dev/null +++ b/tools/lib/UnreadableSkill.php @@ -0,0 +1,17 @@ + { + const name = title.trim().replace(/^["']|["']$/g, ""); + const hash = hashOf(fence.trim()); + const built = examples[hash]; + + // An example with nothing built for it is left as its fence: the block says what + // it was, and a reference to a file that is not there is worse than the source. + return built ? expansion(name, built, fence.trim(), hash) : fence.trim(); + }, + ) + .trim(); +} + +/** + * One example, as a named entry. + * + * @param {{document: string|null, previews: list}} built What was built. + */ +function expansion(name, built, fence, hash) { + // The heading is the tag's title, and the directory is named from what was built. + // They are usually the same, because the title travels from the tag to the manifest; + // they differ when the tag carries no title and the fence does, and the directory + // follows the manifest because that is the one thing both sides can read. + const titled = name !== "" ? name : (built.name ?? ""); + + const directory = `examples/site/${slug(built.name ?? titled)}-${hash}`; + + const lines = [`#### Example: ${titled}`, ""]; + + lines.push(`Source: \`${directory}/example.md\``); + + if (built.document) { + lines.push(`Built: \`${directory}/example.docx\``); + } + + if (built.previews.length > 0) { + const images = built.previews.map((preview) => `\`${directory}/${basename(preview)}\``); + + lines.push(`Images: ${images.join(", ")}`); + } else { + lines.push( + "Images: none — the machine that built this page had no WordCraft or DeckCraft to draw it with.", + ); + } + + lines.push("", fence, ""); + + return lines.join("\n"); +} + +/** + * A name a reader can recognise, from a title. + */ +export function slug(name) { + // The run of characters that are not letters or digits has already collapsed into one + // dash, so there is one dash at each end at most and one is enough to take it off. + return name + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-/, "") + .replace(/-$/, "") + .slice(0, 48); +} + +/** + * The last part of a path. + */ +function basename(path) { + return path.split("/").pop() ?? path; +} diff --git a/tools/write-skill.js b/tools/write-skill.js new file mode 100644 index 0000000..2d793db --- /dev/null +++ b/tools/write-skill.js @@ -0,0 +1,276 @@ +import { cp, mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import { existsSync, readFileSync } from "node:fs"; +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { fileURLToPath } from "node:url"; +import { dirname, join, relative, resolve } from "node:path"; + +import { expandExamples, slug } from "./markdown.js"; + +const here = dirname(fileURLToPath(import.meta.url)); + +/** + * The tools the library ships, which the bundle carries unchanged. + * + * They are copied rather than rewritten, and the checker says so: a copy that has drifted + * is a copy that is wrong, and a script the bundle carries is a script somebody has to + * keep working. + */ +const LIBRARY_TOOLS = [ + "render-document.php", + "render-with-word.applescript", + "render-with-word.ps1", + "package-as-document.sh", + "pdf-is-complete.sh", +]; + +/** + * Where the library is checked out beside this one. + * + * The site's build resolves the libraries through path repositories, so the sibling is + * there or the build never started. It is named here rather than read out of the manifest + * because the manifest says which *version* of the library to use and this says where it + * is on this machine. + */ +const LIBRARY = join(here, "..", "..", "markdown-word"); + +/** + * Where the archive is made by. + * + * The absolute path rather than the name, because a command looked up on the path is a + * command the path decides, and the path is not the build's to decide. + */ +const ZIP = "/usr/bin/zip"; + +/** + * Writes the skill: this site's documentation as a directory an agent can read. + * + * The bundle is a copy, and it is made by the same build that makes the site — so it + * cannot say anything the site does not say, and it cannot be out of date in a way the site + * is not. + * + * Four things are in it: the pages of the site, the examples the site built, the examples + * the library ships, and the tools that render a document so that it can be looked at. + */ +export async function writeSkill({ dir }) { + const manifest = JSON.parse( + readFileSync(join(here, "..", "skill", "mdword", "pages.json"), "utf8"), + ); + const examples = readManifest(); + + const bundle = join(dir.output, "skills", "mdword"); + + await rm(bundle, { recursive: true, force: true }); + await mkdir(bundle, { recursive: true }); + + // The skill itself, and the scripts it points at. + await cp(join(here, "..", "skill", "mdword", "SKILL.md"), join(bundle, "SKILL.md")); + await mkdir(join(bundle, "scripts"), { recursive: true }); + + await Promise.all( + (await readdir(join(here, "..", "skill", "mdword", "scripts"))).map((script) => + cp( + join(here, "..", "skill", "mdword", "scripts", script), + join(bundle, "scripts", script), + ), + ), + ); + + await Promise.all( + LIBRARY_TOOLS.map(async (tool) => { + const from = join(LIBRARY, "tools", tool); + + if (! existsSync(from)) { + throw new Error( + `The tool "${tool}" is not there. The library is checked out beside this one, and it is ` + + "where the bundle's scripts come from.", + ); + } + + await cp(from, join(bundle, "scripts", tool)); + }), + ); + + // The pages, one file each, with the frontmatter off. + await mkdir(join(bundle, "references"), { recursive: true }); + + await Promise.all( + (manifest.references ?? []).map(async (reference) => { + const page = join(here, "..", "content", reference.page); + + if (! existsSync(page)) { + throw new Error(`The page "${reference.page}" is not there, and the bundle says it carries it.`); + } + + const source = readFileSync(page, "utf8"); + + await writeFile( + join(bundle, "references", reference.file), + `${expandExamples(source, examples)}\n`, + ); + }), + ); + + // The examples the site built, one directory each. + await mkdir(join(bundle, "examples", "site"), { recursive: true }); + + await Promise.all( + Object.entries(examples).map(async ([hash, built]) => { + const from = join(here, "..", "_examples", built.language ?? "word", hash); + + if (! existsSync(from)) { + return; + } + + await cp(from, join(bundle, "examples", "site", directoryFor(built.name ?? hash, hash)), { + recursive: true, + }); + }), + ); + + // The examples the library ships: the catalogue of what it can do. + const libraryExamples = join(LIBRARY, "examples"); + + if (! existsSync(libraryExamples)) { + throw new Error( + "The library's examples are not there. The library is checked out beside this one, and they " + + "are what the bundle's own catalogue is.", + ); + } + + await cp(libraryExamples, join(bundle, "examples", "library"), { + recursive: true, + filter: (from) => !from.includes("/out/") && !from.endsWith(".DS_Store"), + }); + + const version = await versionOf(bundle); + + await writeFile(join(bundle, "VERSION"), `${version}\n`); + + // The catalogue OpenCode reads over the network, and the archive everybody else unpacks. + const files = await listFiles(bundle); + + await writeFile( + join(dir.output, "skills", "index.json"), + `${JSON.stringify( + { + skills: [ + { + name: "mdword", + version, + files: files.map((file) => relative(bundle, file).replaceAll("\\", "/")), + }, + ], + }, + null, + 2, + )}\n`, + ); + + await archive(bundle, join(dir.output, "skills", "mdword.zip")); +} + +/** + * What was built for each example, keyed by the hash of the fence it was written as. + */ +function readManifest() { + try { + return JSON.parse(readFileSync(join(here, "..", "_data", "examples.json"), "utf8")); + } catch { + return {}; + } +} + +/** + * The directory an example is carried in: a name a reader can recognise, and the hash that + * makes it unique. + */ +function directoryFor(name, hash) { + return `${slug(name)}-${hash}`; +} + +/** + * Every file in the bundle, in the order they are listed. + * + * @returns {Promise} + */ +async function listFiles(directory) { + const entries = await readdir(directory, { withFileTypes: true }); + + const found = await Promise.all( + entries.map(async (entry) => { + const path = join(directory, entry.name); + + return entry.isDirectory() ? listFiles(path) : [path]; + }), + ); + + const every = found.flat(); + + // The comparison is written out rather than left to the default, because the default is + // a comparison by code unit and this is a comparison by name. + return every.sort(byName); +} + +/** + * How two names stand to one another. + */ +function byName(left, right) { + if (left < right) { + return -1; + } + + if (left > right) { + return 1; + } + + return 0; +} + +/** + * The version of the bundle: the day it was made, and what it held. + */ +async function versionOf(bundle) { + const digest = createHash("sha256"); + const files = await listFiles(bundle); + + // The files are read together and hashed in order, because the digest is of the bundle + // and not of the order the files happened to be read in. + const contents = await Promise.all(files.map((file) => readFile(file))); + + files.forEach((file, index) => { + digest.update(relative(bundle, file)); + digest.update(contents[index]); + }); + + const day = new Date().toISOString().slice(0, 10).replaceAll("-", "."); + + return `${day}-${digest.digest("hex").slice(0, 6)}`; +} + +/** + * The archive everybody but OpenCode unpacks. + * + * The `zip` command rather than a library, because there is no zip library in this project + * and there is a `zip` on every machine the build runs on. A build that cannot make the + * archive says so rather than publishing a bundle nobody can install. + */ +async function archive(bundle, target) { + await rm(target, { force: true }); + + try { + // The target is absolute because the command runs inside the bundle: a relative one + // would be written into the directory being zipped, and the archive would then be + // trying to include itself. + execFileSync(ZIP, ["-r", "-q", resolve(target), "."], { cwd: bundle }); + } catch (error) { + throw new Error( + `The archive could not be made: ${error.message}. The build needs the \`zip\` command, and ` + + "every machine the site is built on has it.", + ); + } + + if (! existsSync(target)) { + throw new Error("The archive was not written, so the bundle cannot be installed."); + } +} diff --git a/tools/yaml.js b/tools/yaml.js new file mode 100644 index 0000000..09c1526 --- /dev/null +++ b/tools/yaml.js @@ -0,0 +1,10 @@ +import { createRequire } from "node:module"; + +// js-yaml has moved between ESM and CommonMark across majors, so going through +// `createRequire` loads whichever this project has installed rather than betting on an +// import shape. +const yaml = createRequire(import.meta.url)("js-yaml"); + +// js-yaml renamed `parse` to `load` in v5 and moved between import systems at the same +// time, so both names are asked for rather than betting on one. +export const parseYaml = yaml.parse ?? yaml.load;