Convert page Markdown responses from the rendered HTML

apps/core/templates/components/feedback.html

@@ -1,6 +1,6 @@
 {% load i18n %}
 
-<div class="feedback">
+<div class="feedback" data-markdown-ignore>
     {% csrf_token %}
     <div class="feedback__container" data-feedback-container>
         <h2 class="feedback__heading" dir="auto">{% trans 'Was this page helpful?' %}</h2>

apps/core/templates/components/language_selector.html

@@ -1,6 +1,6 @@
 {% load static wagtailcore_tags i18n core_tags %}
 
-<div class="language-selector">
+<div class="language-selector" data-markdown-ignore>
     <div class="language-selector__container">
         {% if page %}
         <button class="language-selector__toggle dropdown-toggle" type="button" data-bs-toggle="dropdown" aria-expanded="false">

apps/llms_txt/jinja2/llms_txt/frontmatter.md.jinja

@@ -0,0 +1,12 @@
+{% if frontmatter -%}
+---
+title: {{ (page.seo_title if page.seo_title else page.title)|tojson }}
+{% if page.search_description -%}
+description: {{ page.search_description|tojson }}
+{% endif -%}
+{% if page.last_published_at -%}
+last_modified: {{ page.last_published_at.date().isoformat()|tojson }}
+{% endif -%}
+---
+
+{% endif -%}

apps/llms_txt/jinja2/llms_txt/page.md.jinja

@@ -1,16 +1,4 @@
-{% if frontmatter -%}
----
-title: {{ (page.seo_title if page.seo_title else page.title)|tojson }}
-{% if page.search_description -%}
-description: {{ page.search_description|tojson }}
-{% endif -%}
-{% if page.last_published_at -%}
-last_modified: {{ page.last_published_at.date().isoformat()|tojson }}
-{% endif -%}
----
-
-{% endif -%}
-# {{ page.seo_title if page.seo_title else page.title }}
+{% include "llms_txt/frontmatter.md.jinja" %}# {{ page.seo_title if page.seo_title else page.title }}
 
 Page URL: {{ page.full_url }}
 {% if page.search_description %}

apps/llms_txt/jinja2/llms_txt/page_from_html.md.jinja

@@ -0,0 +1,6 @@
+{#- A page converted from its rendered HTML: see MarkdownRouteMixin.to_markdown_from_html. -#}
+{% include "llms_txt/frontmatter.md.jinja" %}Page URL: {{ page.full_url }}
+{% if page.search_description %}
+> {{ page.search_description }}
+{% endif %}
+{{ body }}

apps/llms_txt/mixins.py

@@ -43,8 +43,34 @@ def to_markdown(self, request=None, frontmatter=False):
         }
         return template.render(context, request)
 
+    def to_markdown_from_html(self, request):
+        """
+        Render this page as Markdown, converted from the page's own HTML.
+
+        Unlike `to_markdown`, this keeps everything visitors see around the
+        content, like the "In this section" lists of section pages and links
+        to llms.txt. The navigation menu is left out by the converter, and
+        elements with `data-markdown-ignore` are too.
+        """
+        # Wagtail's converter imports rich text features, which need the app
+        # registry, and models import this module before it's ready.
+        from wagtail.admin.rich_text.converters.html_markdown import (
+            HTMLToMarkdownConverter,
+        )
+
+        response = RoutablePageMixin.serve(self, request)
+        if hasattr(response, "render"):
+            response.render()
+        converter = HTMLToMarkdownConverter(
+            base_url=self.full_url, content_selector="body"
+        )
+        body = converter.convert(response.content.decode(response.charset))
+        template = loader.get_template("llms_txt/page_from_html.md.jinja")
+        context = {"page": self, "request": request, "frontmatter": True, "body": body}
+        return template.render(context, request)
+
     def markdown_response(self, request):
-        content = self.to_markdown(request, frontmatter=True)
+        content = self.to_markdown_from_html(request)
         response = HttpResponse(content, content_type=MARKDOWN_CONTENT_TYPE)
         response["x-markdown-tokens"] = estimate_tokens(content)
         return response

apps/llms_txt/tests/test_autoescape.py

@@ -31,7 +31,13 @@ def test_page_markdown(self):
         response = self.client.get(
             self.content_page.url, headers={"accept": "text/markdown"}
         )
-        self.assertNotEscaped(response.content.decode())
+        content = response.content.decode()
+        # Converted from the page's HTML, the title gets Markdown escaping,
+        # where `\<` reads as `<`, but no HTML entities.
+        self.assertIn("# Bread & Butter \\<3", content)
+        self.assertIn(DESCRIPTION, content)
+        for entity in ("&amp;", "&lt;", "&#39;", "&#34;", "&quot;"):
+            self.assertNotIn(entity, content)
 
     def test_llms_txt(self):
         response = self.client.get("/llms.txt")

apps/llms_txt/tests/test_mixins.py

@@ -1,6 +1,6 @@
 from django.test import TestCase
 
-from apps.core.factories import ContentPageFactory
+from apps.core.factories import ContentPageFactory, HomePageFactory
 
 
 class TestMarkdownRouteMixin(TestCase):
@@ -89,3 +89,52 @@ def test_homepage_supports_markdown(self):
         home = self.content_page.get_parent().specific
         response = self.get("text/markdown", path=home.url)
         self.assertEqual(response["Content-Type"], "text/markdown;charset=utf-8")
+
+
+class TestMarkdownFromHtml(TestCase):
+    """
+    Page Markdown responses are converted from the page's rendered HTML, so
+    agents see what visitors see around the content.
+    """
+
+    def setUp(self):
+        self.home_page = HomePageFactory()
+        self.section = ContentPageFactory(parent=self.home_page, title="Releases")
+        self.child = ContentPageFactory(parent=self.section, title="New in Wagtail 8.1")
+        self.menu_page = ContentPageFactory(
+            parent=self.home_page, title="Manage images", show_in_menus=True
+        )
+
+    def get_markdown(self, page):
+        response = self.client.get(page.url, headers={"accept": "text/markdown"})
+        return response.content.decode()
+
+    def test_section_pages_list_their_pages(self):
+        markdown = self.get_markdown(self.section)
+        self.assertIn("## In this section", markdown)
+        self.assertIn(f"[New in Wagtail 8.1]({self.child.full_url})", markdown)
+
+    def test_links_to_llms_txt(self):
+        markdown = self.get_markdown(self.section)
+        self.assertIn("[View llms.txt](http://localhost/llms.txt)", markdown)
+
+    def test_keeps_the_header(self):
+        markdown = self.get_markdown(self.section)
+        self.assertTrue(markdown.startswith("---\n"))
+        self.assertIn(f"Page URL: {self.section.full_url}", markdown)
+
+    def test_leaves_out_the_feedback_form(self):
+        markdown = self.get_markdown(self.section)
+        self.assertNotIn("Was this page helpful?", markdown)
+
+    def test_leaves_out_the_menu(self):
+        html = self.client.get(self.section.url).content.decode()
+        self.assertIn(f'href="{self.menu_page.url}"', html)
+        markdown = self.get_markdown(self.section)
+        self.assertNotIn(self.menu_page.full_url, markdown)
+
+    def test_to_markdown_is_unchanged(self):
+        """llms-full.txt and the MCP server keep using the page's content only."""
+        markdown = self.section.to_markdown()
+        self.assertNotIn("In this section", markdown)
+        self.assertNotIn("llms.txt", markdown)