From de1be492244eab01f1b18541567e89f5638ac070 Mon Sep 17 00:00:00 2001
From: Albina Starykova <albina.starikova@gmail.com>
Date: Wed, 28 Oct 2026 10:46:48 +0200
Subject: [PATCH] Convert page Markdown responses from the rendered HTML

---
 apps/core/templates/components/feedback.html  |  2 +-
 .../components/language_selector.html         |  2 +-
 .../jinja2/llms_txt/frontmatter.md.jinja      | 12 +++++
 apps/llms_txt/jinja2/llms_txt/page.md.jinja   | 14 +----
 .../jinja2/llms_txt/page_from_html.md.jinja   |  6 +++
 apps/llms_txt/mixins.py                       | 28 +++++++++-
 apps/llms_txt/tests/test_autoescape.py        |  8 ++-
 apps/llms_txt/tests/test_mixins.py            | 51 ++++++++++++++++++-
 8 files changed, 105 insertions(+), 18 deletions(-)
 create mode 100644 apps/llms_txt/jinja2/llms_txt/frontmatter.md.jinja
 create mode 100644 apps/llms_txt/jinja2/llms_txt/page_from_html.md.jinja

diff --git a/apps/core/templates/components/feedback.html b/apps/core/templates/components/feedback.html
index 14cf82df..ce11121b 100644
--- a/apps/core/templates/components/feedback.html
+++ b/apps/core/templates/components/feedback.html
@@ -1,6 +1,6 @@
 {% load i18n %}
 
-<div class="feedback">
+<div class="feedback" data-markdown-ignore>
     {% csrf_token %}
     <div class="feedback__container" data-feedback-container>
         <h2 class="feedback__heading" dir="auto">{% trans 'Was this page helpful?' %}</h2>
diff --git a/apps/core/templates/components/language_selector.html b/apps/core/templates/components/language_selector.html
index 98f39d47..3d2662fa 100644
--- a/apps/core/templates/components/language_selector.html
+++ b/apps/core/templates/components/language_selector.html
@@ -1,6 +1,6 @@
 {% load static wagtailcore_tags i18n core_tags %}
 
-<div class="language-selector">
+<div class="language-selector" data-markdown-ignore>
     <div class="language-selector__container">
         {% if page %}
         <button class="language-selector__toggle dropdown-toggle" type="button" data-bs-toggle="dropdown" aria-expanded="false">
diff --git a/apps/llms_txt/jinja2/llms_txt/frontmatter.md.jinja b/apps/llms_txt/jinja2/llms_txt/frontmatter.md.jinja
new file mode 100644
index 00000000..7c832faf
--- /dev/null
+++ b/apps/llms_txt/jinja2/llms_txt/frontmatter.md.jinja
@@ -0,0 +1,12 @@
+{% if frontmatter -%}
+---
+title: {{ (page.seo_title if page.seo_title else page.title)|tojson }}
+{% if page.search_description -%}
+description: {{ page.search_description|tojson }}
+{% endif -%}
+{% if page.last_published_at -%}
+last_modified: {{ page.last_published_at.date().isoformat()|tojson }}
+{% endif -%}
+---
+
+{% endif -%}
diff --git a/apps/llms_txt/jinja2/llms_txt/page.md.jinja b/apps/llms_txt/jinja2/llms_txt/page.md.jinja
index f901041e..1485fa04 100644
--- a/apps/llms_txt/jinja2/llms_txt/page.md.jinja
+++ b/apps/llms_txt/jinja2/llms_txt/page.md.jinja
@@ -1,16 +1,4 @@
-{% if frontmatter -%}
----
-title: {{ (page.seo_title if page.seo_title else page.title)|tojson }}
-{% if page.search_description -%}
-description: {{ page.search_description|tojson }}
-{% endif -%}
-{% if page.last_published_at -%}
-last_modified: {{ page.last_published_at.date().isoformat()|tojson }}
-{% endif -%}
----
-
-{% endif -%}
-# {{ page.seo_title if page.seo_title else page.title }}
+{% include "llms_txt/frontmatter.md.jinja" %}# {{ page.seo_title if page.seo_title else page.title }}
 
 Page URL: {{ page.full_url }}
 {% if page.search_description %}
diff --git a/apps/llms_txt/jinja2/llms_txt/page_from_html.md.jinja b/apps/llms_txt/jinja2/llms_txt/page_from_html.md.jinja
new file mode 100644
index 00000000..8c122626
--- /dev/null
+++ b/apps/llms_txt/jinja2/llms_txt/page_from_html.md.jinja
@@ -0,0 +1,6 @@
+{#- A page converted from its rendered HTML: see MarkdownRouteMixin.to_markdown_from_html. -#}
+{% include "llms_txt/frontmatter.md.jinja" %}Page URL: {{ page.full_url }}
+{% if page.search_description %}
+> {{ page.search_description }}
+{% endif %}
+{{ body }}
diff --git a/apps/llms_txt/mixins.py b/apps/llms_txt/mixins.py
index c25206f9..511fe3e7 100644
--- a/apps/llms_txt/mixins.py
+++ b/apps/llms_txt/mixins.py
@@ -43,8 +43,34 @@ def to_markdown(self, request=None, frontmatter=False):
         }
         return template.render(context, request)
 
+    def to_markdown_from_html(self, request):
+        """
+        Render this page as Markdown, converted from the page's own HTML.
+
+        Unlike `to_markdown`, this keeps everything visitors see around the
+        content, like the "In this section" lists of section pages and links
+        to llms.txt. The navigation menu is left out by the converter, and
+        elements with `data-markdown-ignore` are too.
+        """
+        # Wagtail's converter imports rich text features, which need the app
+        # registry, and models import this module before it's ready.
+        from wagtail.admin.rich_text.converters.html_markdown import (
+            HTMLToMarkdownConverter,
+        )
+
+        response = RoutablePageMixin.serve(self, request)
+        if hasattr(response, "render"):
+            response.render()
+        converter = HTMLToMarkdownConverter(
+            base_url=self.full_url, content_selector="body"
+        )
+        body = converter.convert(response.content.decode(response.charset))
+        template = loader.get_template("llms_txt/page_from_html.md.jinja")
+        context = {"page": self, "request": request, "frontmatter": True, "body": body}
+        return template.render(context, request)
+
     def markdown_response(self, request):
-        content = self.to_markdown(request, frontmatter=True)
+        content = self.to_markdown_from_html(request)
         response = HttpResponse(content, content_type=MARKDOWN_CONTENT_TYPE)
         response["x-markdown-tokens"] = estimate_tokens(content)
         return response
diff --git a/apps/llms_txt/tests/test_autoescape.py b/apps/llms_txt/tests/test_autoescape.py
index 4506b151..ae8b95cb 100644
--- a/apps/llms_txt/tests/test_autoescape.py
+++ b/apps/llms_txt/tests/test_autoescape.py
@@ -31,7 +31,13 @@ def test_page_markdown(self):
         response = self.client.get(
             self.content_page.url, headers={"accept": "text/markdown"}
         )
-        self.assertNotEscaped(response.content.decode())
+        content = response.content.decode()
+        # Converted from the page's HTML, the title gets Markdown escaping,
+        # where `\<` reads as `<`, but no HTML entities.
+        self.assertIn("# Bread & Butter \\<3", content)
+        self.assertIn(DESCRIPTION, content)
+        for entity in ("&amp;", "&lt;", "&#39;", "&#34;", "&quot;"):
+            self.assertNotIn(entity, content)
 
     def test_llms_txt(self):
         response = self.client.get("/llms.txt")
diff --git a/apps/llms_txt/tests/test_mixins.py b/apps/llms_txt/tests/test_mixins.py
index 46897427..a514ad79 100644
--- a/apps/llms_txt/tests/test_mixins.py
+++ b/apps/llms_txt/tests/test_mixins.py
@@ -1,6 +1,6 @@
 from django.test import TestCase
 
-from apps.core.factories import ContentPageFactory
+from apps.core.factories import ContentPageFactory, HomePageFactory
 
 
 class TestMarkdownRouteMixin(TestCase):
@@ -89,3 +89,52 @@ def test_homepage_supports_markdown(self):
         home = self.content_page.get_parent().specific
         response = self.get("text/markdown", path=home.url)
         self.assertEqual(response["Content-Type"], "text/markdown;charset=utf-8")
+
+
+class TestMarkdownFromHtml(TestCase):
+    """
+    Page Markdown responses are converted from the page's rendered HTML, so
+    agents see what visitors see around the content.
+    """
+
+    def setUp(self):
+        self.home_page = HomePageFactory()
+        self.section = ContentPageFactory(parent=self.home_page, title="Releases")
+        self.child = ContentPageFactory(parent=self.section, title="New in Wagtail 8.1")
+        self.menu_page = ContentPageFactory(
+            parent=self.home_page, title="Manage images", show_in_menus=True
+        )
+
+    def get_markdown(self, page):
+        response = self.client.get(page.url, headers={"accept": "text/markdown"})
+        return response.content.decode()
+
+    def test_section_pages_list_their_pages(self):
+        markdown = self.get_markdown(self.section)
+        self.assertIn("## In this section", markdown)
+        self.assertIn(f"[New in Wagtail 8.1]({self.child.full_url})", markdown)
+
+    def test_links_to_llms_txt(self):
+        markdown = self.get_markdown(self.section)
+        self.assertIn("[View llms.txt](http://localhost/llms.txt)", markdown)
+
+    def test_keeps_the_header(self):
+        markdown = self.get_markdown(self.section)
+        self.assertTrue(markdown.startswith("---\n"))
+        self.assertIn(f"Page URL: {self.section.full_url}", markdown)
+
+    def test_leaves_out_the_feedback_form(self):
+        markdown = self.get_markdown(self.section)
+        self.assertNotIn("Was this page helpful?", markdown)
+
+    def test_leaves_out_the_menu(self):
+        html = self.client.get(self.section.url).content.decode()
+        self.assertIn(f'href="{self.menu_page.url}"', html)
+        markdown = self.get_markdown(self.section)
+        self.assertNotIn(self.menu_page.full_url, markdown)
+
+    def test_to_markdown_is_unchanged(self):
+        """llms-full.txt and the MCP server keep using the page's content only."""
+        markdown = self.section.to_markdown()
+        self.assertNotIn("In this section", markdown)
+        self.assertNotIn("llms.txt", markdown)