Add clean markdown generation for LLM-friendly page content #723

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged

nearestnabors merged 9 commits into main from feat/clean-markdown-for-llms

Feb 6, 2026

+175,755 −74

.github/workflows/generate-markdown.yml

-Original file line number
+Diff line change
@@ -0,0 +1,88 @@
+    name: Generate Clean Markdown
+    on:
+      workflow_dispatch:
+      pull_request:
+        types: [opened, synchronize, reopened]
+        paths:
+          - "app/**/*.mdx"
+          - "app/**/_meta.tsx"
+          - "scripts/generate-clean-markdown.ts"
+    permissions:
+      contents: write
+      pull-requests: write
+    jobs:
+      generate-markdown:
+        name: Generate Clean Markdown
+        runs-on: ubuntu-latest
+        permissions:
+          contents: write
+          pull-requests: write
+        steps:
+          - name: Use Node.js
+            uses: actions/setup-node@v4
+            with:
+              node-version: "22.x"
+          - name: Checkout code
+            uses: actions/checkout@v4
+            with:
+              ref: ${{ github.event.pull_request.head.ref || github.ref }}
+              token: ${{ secrets.DOCS_PUBLISHABLE_GH_TOKEN }}
+          - name: Install pnpm
+            run: npm install -g pnpm
+          - name: Install dependencies
+            run: pnpm install
+          - name: Build Next.js
+            run: pnpm build
+            env:
+              # Skip postbuild to avoid circular dependency
+              SKIP_POSTBUILD: "true"
+          - name: Generate clean markdown
+            run: pnpm generate:markdown
+          - name: Check for changes
+            id: check-changes
+            run: |
+              if [ -n "$(git status --porcelain public/_markdown/)" ]; then
+                echo "has_changes=true" >> $GITHUB_OUTPUT
+              else
+                echo "has_changes=false" >> $GITHUB_OUTPUT
+              fi
+          - name: Commit changes to PR
+            if: steps.check-changes.outputs.has_changes == 'true' && github.event_name == 'pull_request'
+            run: |
+              git config user.name "github-actions[bot]"
+              git config user.email "github-actions[bot]@users.noreply.github.com"
+              git add public/_markdown/
+              git commit -m "Regenerate clean markdown files"
+              git push
+          - name: Create Pull Request (for manual runs)
+            if: steps.check-changes.outputs.has_changes == 'true' && github.event_name != 'pull_request'
+            id: cpr
+            uses: peter-evans/create-pull-request@v7
+            with:
+              token: ${{ secrets.DOCS_PUBLISHABLE_GH_TOKEN }}
+              commit-message: Regenerate clean markdown files
+              branch: auto-update-clean-markdown
+              delete-branch: true
+              title: "Regenerate clean markdown files"
+              reviewers: >
+                evantahler
+                torresmateo
+          - name: Enable Pull Request Automerge
+            if: steps.check-changes.outputs.has_changes == 'true' && github.event_name != 'pull_request'
+            run: gh pr merge --squash --auto ${{ steps.cpr.outputs.pull-request-number }}
+            env:
+              GH_TOKEN: ${{ secrets.DOCS_PUBLISHABLE_GH_TOKEN }}

app/_components/copy-page-override.tsx

-Original file line number
+Diff line change
@@ -0,0 +1,101 @@
+    "use client";
+    import { usePathname } from "next/navigation";
+    import { useCallback, useEffect } from "react";
+    const COPY_FEEDBACK_DELAY_MS = 2000;
+    const COPY_BUTTON_TEXT = "Copy page";
+    const COPIED_TEXT = "Copied";
+    const DROPDOWN_IDENTIFIER = "Markdown for LLMs";
+    /**
+     * This component overrides the default nextra-theme-docs "Copy page" button behavior
+     * to fetch clean markdown from our API instead of copying raw MDX source.
+     */
+    export function CopyPageOverride() {
+      const pathname = usePathname();
+      const fetchAndCopyMarkdown = useCallback(async (): Promise<boolean> => {
+        try {
+          const markdownUrl = `/api/markdown${pathname}.md`;
+          const response = await fetch(markdownUrl);
+          if (!response.ok) {
+            throw new Error(`Failed to fetch markdown: ${response.status}`);
+          }
+          const markdown = await response.text();
+          await navigator.clipboard.writeText(markdown);
+          return true;
+        } catch {
+          return false;
+        }
+      }, [pathname]);
+      useEffect(() => {
+        const isCopyButton = (button: HTMLButtonElement): boolean => {
+          const text = button.textContent || "";
+          return text.includes(COPY_BUTTON_TEXT) || text.includes(COPIED_TEXT);
+        };
+        const updateButtonFeedback = (button: HTMLButtonElement): void => {
+          const textNodes = button.querySelectorAll("*");
+          for (const node of textNodes) {
+            if (node.textContent === COPY_BUTTON_TEXT) {
+              node.textContent = COPIED_TEXT;
+              setTimeout(() => {
+                node.textContent = COPY_BUTTON_TEXT;
+              }, COPY_FEEDBACK_DELAY_MS);
+              return;
+            }
+          }
+        };
+        const handleButtonClick = async (event: MouseEvent): Promise<void> => {
+          const target = event.target as HTMLElement;
+          const button = target.closest("button") as HTMLButtonElement | null;
+          if (!(button && isCopyButton(button))) {
+            return;
+          }
+          event.preventDefault();
+          event.stopPropagation();
+          const success = await fetchAndCopyMarkdown();
+          if (success) {
+            updateButtonFeedback(button);
+          }
+        };
+        const handleDropdownClick = async (event: MouseEvent): Promise<void> => {
+          const target = event.target as HTMLElement;
+          const option = target.closest('[role="option"]');
+          const optionText = option?.textContent || "";
+          const isDropdownCopyOption =
+            optionText.includes(COPY_BUTTON_TEXT) &&
+            optionText.includes(DROPDOWN_IDENTIFIER);
+          if (!isDropdownCopyOption) {
+            return;
+          }
+          event.preventDefault();
+          event.stopPropagation();
+          await fetchAndCopyMarkdown();
+          document.body.click();
+        };
+        document.addEventListener("click", handleButtonClick, true);
+        document.addEventListener("click", handleDropdownClick, true);
+        return () => {
+          document.removeEventListener("click", handleButtonClick, true);
+          document.removeEventListener("click", handleDropdownClick, true);
+        };
+      }, [fetchAndCopyMarkdown]);
+      return null;
+    }

app/_components/custom-layout.tsx

-Original file line number
+Diff line change
@@ -1,4 +1,5 @@
     import type React from "react";
+    import { CopyPageOverride } from "@/app/_components/copy-page-override";
     import { PlaceholderReplacer } from "@/app/_components/placeholder-replacer";
     import { OrySessionProvider } from "@/app/_lib/ory-session-context";
@@ Expand All @@
     }) => (
       <OrySessionProvider>
         <PlaceholderReplacer />
+        <CopyPageOverride />
         <main className="custom-main">{children}</main>
       </OrySessionProvider>
     );
@@ Expand Down @@

app/api/markdown/[[...slug]]/route.ts

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -7,6 +7,9 @@ export const dynamic = "force-dynamic";
  
    // Regex pattern for removing .md extension

    const MD_EXTENSION_REGEX = /\.md$/;

    // Directory containing pre-generated clean markdown files

    const CLEAN_MARKDOWN_DIR = join(process.cwd(), "public", "_markdown");

    export async function GET(

      request: NextRequest,

      _context: { params: Promise<{ slug?: string[] }> }

    @@ -17,28 +20,48 @@ export async function GET(
  
        // Remove /api/markdown prefix to get the original path

        const originalPath = url.pathname.replace("/api/markdown", "");

        // Remove .md extension

        // Remove .md extension if present

        const pathWithoutMd = originalPath.replace(MD_EXTENSION_REGEX, "");

        // Map URL to file path

        // Try clean markdown first (preferred)

        // e.g., /en/home/quickstart -> public/_markdown/en/home/quickstart.md

        const cleanMarkdownPath = join(CLEAN_MARKDOWN_DIR, `${pathWithoutMd}.md`);

        try {

          await access(cleanMarkdownPath);

          const content = await readFile(cleanMarkdownPath, "utf-8");

          return new NextResponse(content, {

            status: 200,

            headers: {

              "Content-Type": "text/plain; charset=utf-8",

              "Content-Disposition": "inline",

              "Cache-Control": "public, max-age=3600", // Cache for 1 hour

            },

          });

        } catch {

          // Clean markdown not found, fall back to raw MDX

        }

        // Fallback: serve raw MDX (for backwards compatibility or if clean files not generated)

        // e.g., /en/home/quickstart -> app/en/home/quickstart/page.mdx

        const filePath = join(process.cwd(), "app", `${pathWithoutMd}/page.mdx`);

        const rawMdxPath = join(process.cwd(), "app", `${pathWithoutMd}/page.mdx`);

        // Check if file exists

        try {

          await access(filePath);

          await access(rawMdxPath);

        } catch {

          return new NextResponse("Markdown file not found", { status: 404 });

        }

        const content = await readFile(filePath, "utf-8");

        const content = await readFile(rawMdxPath, "utf-8");

        // Return the raw markdown with proper headers

        // Return the raw MDX with a warning header

        return new NextResponse(content, {

          status: 200,

          headers: {

            "Content-Type": "text/plain; charset=utf-8",

            "Content-Disposition": "inline",

            "X-Content-Source": "raw-mdx", // Indicate this is raw MDX, not clean markdown

          },

        });

      } catch (error) {

next-env.d.ts

-Original file line number
+Diff line change
@@ -1,6 +1,6 @@
     /// <reference types="next" />
     /// <reference types="next/image-types/global" />
-    import "./.next/dev/types/routes.d.ts";
+    import "./.next/types/routes.d.ts";
     // NOTE: This file should not be edited
     // see https://nextjs.org/docs/app/api-reference/config/typescript for more information.

package.json

-Original file line number
+Diff line change
@@ Expand Up / @@ -10,7 +10,8 @@ @@
         "lint": "pnpm dlx ultracite check",
         "format": "pnpm dlx ultracite fix",
         "prepare": "husky install",
-        "postbuild": "pnpm run custompagefind",
+        "postbuild": "if [ \"$SKIP_POSTBUILD\" != \"true\" ]; then pnpm run generate:markdown && pnpm run custompagefind; fi",
+        "generate:markdown": "pnpm dlx tsx scripts/generate-clean-markdown.ts",
         "translate": "pnpm dlx tsx scripts/i18n-sync/index.ts && pnpm format",
         "sync:metas": "pnpm dlx tsx scripts/sync-metas.ts app/en",
         "llmstxt": "pnpm dlx tsx scripts/generate-llmstxt.ts",
@@ Expand Down Expand Up / @@ -74,6 +75,7 @@ @@
         "@types/react": "19.2.7",
         "@types/react-dom": "19.2.3",
         "@types/react-syntax-highlighter": "15.5.13",
+        "@types/turndown": "^5.0.6",
         "@types/unist": "3.0.3",
         "commander": "14.0.2",
         "dotenv": "^17.2.3",
@@ Expand All / @@ -90,6 +92,7 @@ @@
         "remark": "^15.0.1",
         "remark-rehype": "^11.1.2",
         "tailwindcss": "4.1.16",
+        "turndown": "^7.2.2",
         "typescript": "5.9.3",
         "ultracite": "6.1.0",
         "vitest": "4.0.5",
@@ Expand Down @@

pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Add clean markdown generation for LLM-friendly page content #723

Diff view

Diff view

There are no files selected for viewing

Uh oh!

Uh oh!