diff --git a/.gitignore b/.gitignore index f30fd66606..1f2f58534f 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,5 @@ lychee.toml lychee-report.md .phpunit.result.cache code_samples/_inline_php/ +/build/ +*.egg-info/ diff --git a/hooks.py b/hooks.py index dfdbe4ba70..a88502fba9 100644 --- a/hooks.py +++ b/hooks.py @@ -1,96 +1,5 @@ -""" -MkDocs hooks for Ibexa developer documentation. +"""MkDocs hooks entry point — delegates to the installable llms_txt package.""" -- Keeps the llmstxt plugin's ``sections`` config in sync with the ``nav`` - defined in ``mkdocs.yml``. -- Post-processes the Markdown generated by the llmstxt plugin. +from llms_txt.hooks import on_config, on_page_content -All content transformations live in ``llmstxt_preprocess.py``; this module is -only MkDocs event glue. -""" - -from __future__ import annotations - -import sys -from pathlib import Path, PurePosixPath -from typing import TYPE_CHECKING - -_here = Path(__file__).parent -if str(_here) not in sys.path: - sys.path.insert(0, str(_here)) - -from llmstxt_preprocess import ( - absolutize_image_urls, - editions_from_frontmatter, - expand_macros, - inject_page_metadata, - renumber_ordered_lists, -) -from update_llmstxt_config import convert_nav_to_llmstxt_sections - -if TYPE_CHECKING: - from mkdocs.config.defaults import MkDocsConfig - from mkdocs.structure.pages import Page - - -def on_config(config: "MkDocsConfig") -> None: - """Populate llmstxt sections from nav before the build starts. - - The llmstxt plugin reads ``config.sections`` in its ``on_files`` event, - which fires after ``on_config``, so injecting here is the right place. - """ - nav = config.get("nav") - if not nav: - return - - llmstxt = config["plugins"].get("llmstxt") - if llmstxt is None: - return - - docs_dir = Path(config["docs_dir"]) - llmstxt.config.sections = convert_nav_to_llmstxt_sections(nav, docs_dir) - - -def on_page_content(html: str, *, page: "Page", config: "MkDocsConfig", **kwargs) -> None: - """Reformat the Markdown content generated by the llmstxt plugin for this page. - - Hooks run after plugins for every event, so at this point the llmstxt - plugin has already generated Markdown and stored it in ``_md_pages``. - We reformat here so the plugin writes the corrected content in its own - ``on_post_build`` (which also runs before ours, for the same reason). - """ - llmstxt = config["plugins"].get("llmstxt") - if llmstxt is None: - return - - src_uri = page.file.src_uri - page_info = llmstxt._md_pages.get(src_uri) - if page_info is None: - return - - frontmatter = _read_frontmatter(page, config) - editions = editions_from_frontmatter(frontmatter) - description = expand_macros(str(frontmatter.get("description") or ""), config.get("extra") or {}) - if "[[=" in description: - # Unresolved macros must not leak into the output. - description = "" - content = inject_page_metadata(page_info.content, description, editions) - content = renumber_ordered_lists(content) - # Same base URL and page directory the plugin uses for making link hrefs absolute. - page_dir = PurePosixPath(page.file.dest_uri).parent.as_posix() - content = absolutize_image_urls(content, llmstxt._base_url, page_dir) - if content != page_info.content: - llmstxt._md_pages[src_uri] = page_info._replace(content=content) - - -def _read_frontmatter(page: "Page", config: "MkDocsConfig") -> dict: - """Read the page's YAML frontmatter from its source file.""" - src_path = Path(config["docs_dir"]) / page.file.src_path - try: - from mkdocs.utils import meta as mkdocs_meta - with open(src_path, encoding="utf-8") as f: - raw = f.read() - _, frontmatter = mkdocs_meta.get_data(raw) - return frontmatter - except Exception: - return {} +__all__ = ["on_config", "on_page_content"] diff --git a/llmstxt_preprocess.py b/llmstxt_preprocess.py index f0e3c7c6fb..e4d5b126c9 100644 --- a/llmstxt_preprocess.py +++ b/llmstxt_preprocess.py @@ -1,547 +1,5 @@ -""" -Markdown conversion logic for the llmstxt plugin. +"""mkdocs-llmstxt ``preprocess:`` entry point — delegates to the installable llms_txt package.""" -This module is the single place holding all content transformations used to -produce the Markdown version of the documentation: +from llms_txt.llmstxt_preprocess import preprocess -- ``preprocess(soup, output)`` — HTML-level transforms invoked by the - mkdocs-llmstxt plugin (configured in ``plugins.yml``) before the HTML is - converted to Markdown. -- Markdown post-processing helpers (``renumber_ordered_lists``, - ``inject_edition_badges``, ``editions_from_frontmatter``) applied by - ``hooks.py`` to the Markdown the plugin generated. - -Note: the plugin loads this file by path (``spec_from_file_location``) while -``hooks.py`` imports it as a regular module, so two module objects may coexist. -Keep this module stateless (constants and pure functions only). -""" - -import html as html_module -import re -from urllib.parse import urljoin, urlparse - -from bs4 import BeautifulSoup as Soup, NavigableString - -PILL_CLASS_TO_EDITION = { - "pill--lts-update": "LTS Update", - "pill--experience": "Experience", - "pill--commerce": "Commerce", - "pill--headless": "Headless", - "pill--new-feature": "New feature", - "pill--first-release": "First release", -} - -FRONTMATTER_EDITION_DISPLAY = { - "lts-update": "LTS Update", - "experience": "Experience", - "commerce": "Commerce", - "headless": "Headless", -} - - -def preprocess(soup: Soup, output: str) -> None: - """ - Preprocess HTML to improve markdown conversion. - - Runs with autoclean disabled so we can control the order: - 1. Expand tabbed sets with labels before autoclean removes tabbed-labels. - 2. Run autoclean-equivalent cleanup. - 3. Replace inline edition badge spans with readable text. - 4. Remove release notes filter UI. - 5. Convert card macros to markdown lists. - - Note: frontmatter edition injection is handled in hooks.py on_page_content, - where page.file.src_path is available directly. - """ - _process_tabbed_sets(soup) - _autoclean(soup) - _process_inline_pills(soup) - _process_release_note_tags(soup) - _process_release_note_dates(soup) - _process_release_notes_filters(soup) - _process_cards(soup) - _process_info_tiles(soup) - _process_tables(soup) - _process_admonitions(soup) - - -# --------------------------------------------------------------------------- -# Tables -# --------------------------------------------------------------------------- - -def _process_tables(soup: Soup) -> None: - """Replace ✔ (U+2714) with 'Yes' in table cells for readability.""" - CHECK = "✔" - for cell in soup.find_all(["td", "th"]): - for text_node in cell.find_all(string=lambda s: CHECK in s): - text_node.replace_with(NavigableString(text_node.replace(CHECK, "Yes"))) - - -# --------------------------------------------------------------------------- -# Tabbed sets -# --------------------------------------------------------------------------- - -def _process_tabbed_sets(soup: Soup) -> None: - """Prepend each tab label as bold text before its content block.""" - for tabbed_set in soup.find_all("div", class_="tabbed-set"): - labels_div = tabbed_set.find("div", class_="tabbed-labels") - content_div = tabbed_set.find("div", class_="tabbed-content") - - if not labels_div or not content_div: - tabbed_set.unwrap() - continue - - labels = [label.get_text(strip=True) for label in labels_div.find_all("label")] - blocks = content_div.find_all("div", class_="tabbed-block", recursive=False) - - wrapper = soup.new_tag("div") - for i, block in enumerate(blocks): - if i < len(labels): - label_tag = soup.new_tag("p") - strong = soup.new_tag("strong") - strong.string = labels[i] - label_tag.append(strong) - wrapper.append(label_tag) - for child in list(block.children): - wrapper.append(child.extract()) - - tabbed_set.replace_with(wrapper) - - -# --------------------------------------------------------------------------- -# Autoclean equivalent (mirrors mkdocs-llmstxt autoclean, minus tabbed-labels) -# --------------------------------------------------------------------------- - -def _autoclean(soup: Soup) -> None: - """Replicate the plugin's autoclean so we can run it after tab processing.""" - - # Unwrap links that wrap an image (e.g. lightbox links) so the image - # itself decides its fate below. - for link in soup.find_all("a"): - img = link.find("img") - if img: - link.replace_with(img.extract()) - - # Keep images that have alt text so markdownify emits ![alt](src) - # (URLs are made absolute later by absolutize_image_urls); drop the rest, - # since an image without a description tells an AI agent nothing. - # Done in its own pass: mutating the tree from inside a find_all() predicate - # makes the traversal skip the elements that follow the mutated one. - for img in soup.find_all("img"): - alt = (img.get("alt") or "").strip() - if not alt: - img.decompose() - - def _should_remove(tag) -> bool: - if tag.name == "svg": - return True - classes = tag.get("class") or () - if tag.name == "a" and "headerlink" in classes: - return True - if "twemoji" in classes: - return True - # tabbed-labels are already consumed by _process_tabbed_sets, but - # handle any stragglers defensively. - if "tabbed-labels" in classes: - return True - return False - - for element in soup.find_all(_should_remove): - element.decompose() - - for element in soup.find_all("autoref"): - element.replace_with(NavigableString(element.get_text())) - - # Insert ", " between adjacent elements in table cells. - # html.parser silently drops
(invalid void-closing tag), so adjacent - # blocks have no separator and markdownify concatenates their backticks. - for td in soup.find_all(["td", "th"]): - children = list(td.children) - for i in range(len(children) - 1): - curr = children[i] - nxt = children[i + 1] - if getattr(curr, "name", None) == "code" and getattr(nxt, "name", None) == "code": - curr.insert_after(NavigableString(", ")) - - for element in soup.find_all("div", attrs={"class": "doc-md-description"}): - element.replace_with(NavigableString(element.get_text().strip())) - - for element in soup.find_all("span", attrs={"class": "doc-labels"}): - element.decompose() - - # Flatten line-numbered code blocks to a plain
, keeping the
-    # language-* class (emitted thanks to pygments_lang_class) on the 
,
-    # where markdownify's code_language_callback looks for it.
-    for element in soup.find_all("table", attrs={"class": "highlighttable"}):
-        code_elem = element.find("code")
-        if code_elem:
-            classes = code_elem.get("class") or ()
-            language = next((c for c in classes if c.startswith("language-")), "")
-            attr = f' class="{language}"' if language else ""
-            element.replace_with(
-                Soup(f"{html_module.escape(code_elem.get_text())}
", "html.parser") - ) - -# --------------------------------------------------------------------------- -# Inline edition badge spans (from snippet includes) -# --------------------------------------------------------------------------- - -def _pill_edition(node) -> str: - """Return the edition name of an inline pill span, or '' if not one.""" - if getattr(node, "name", None) != "span": - return "" - classes = node.get("class") or [] - if "pill--inline" not in classes: - return "" - for pill_cls, edition_name in PILL_CLASS_TO_EDITION.items(): - if pill_cls in classes: - return edition_name - return "" - - -def _process_inline_pills(soup: Soup) -> None: - """Replace inline edition pill spans with readable text. - - Consecutive pills (possibly separated by whitespace) are merged into a - single parenthetical, e.g. ' (Experience, Commerce)' instead of - ' (Experience) (Commerce)'. - """ - for span in soup.find_all("span", class_="pill--inline"): - if span.parent is None: # already consumed as part of a previous run - continue - edition = _pill_edition(span) - if not edition: - continue - - # Collect the run of pills that follow, skipping whitespace between them. - editions = [edition] - consumed = [] - node = span.next_sibling - pending_whitespace = [] - while node is not None: - if isinstance(node, NavigableString) and not node.strip(): - pending_whitespace.append(node) - node = node.next_sibling - continue - next_edition = _pill_edition(node) - if not next_edition: - break - editions.append(next_edition) - consumed += pending_whitespace + [node] - pending_whitespace = [] - node = node.next_sibling - - for extra_node in consumed: - extra_node.extract() - span.replace_with(soup.new_string(f" ({', '.join(editions)})")) - - -def _process_release_note_tags(soup: Soup) -> None: - """Append edition labels from release-note__tags divs to their preceding heading. - - Release notes use a
block after each

- containing empty
elements rendered via CSS. - This converts them to a readable parenthetical on the heading, e.g.: - ## Google Gemini connector v5.0.7 (Headless, Experience, LTS Update, New feature) - """ - for tags_div in soup.find_all("div", class_="release-note__tags"): - editions = [] - for pill_div in tags_div.find_all("div"): - classes = pill_div.get("class", []) - for pill_cls, name in PILL_CLASS_TO_EDITION.items(): - if pill_cls in classes: - editions.append(name) - break - - heading = tags_div.find_previous_sibling(["h1", "h2", "h3", "h4"]) - if heading and editions: - # Insert before the permalink anchor so it's part of the heading text - anchor = heading.find("a", class_="headerlink") - label = NavigableString(f" ({', '.join(editions)})") - if anchor: - anchor.insert_before(label) - else: - heading.append(label) - - tags_div.decompose() - - -def _process_release_note_dates(soup: Soup) -> None: - """Prefix release-note dates with 'Release date: ' so the bare date line stays understandable.""" - for date_div in soup.find_all("div", class_="release-note__date"): - date_text = date_div.get_text(strip=True) - if date_text: - date_div.string = f"Release date: {date_text}" - - -def _process_info_tiles(soup: Soup) -> None: - """Simplify info-tile links to a single clean link text. - - Info tiles have a 'Details' label div and a separate content div. - After SVG removal, markdownify produces broken multi-line link text. - Replace the whole content with just the meaningful text. - """ - for tile in soup.find_all("a", class_="info-tile"): - # The label div ("Details") can be discarded - label = tile.find("div", class_="info-tile__details") - if label: - label.decompose() - - # Flatten remaining content to plain text - text = tile.get_text(separator=" ", strip=True) - tile.clear() - tile.append(soup.new_string(text)) - - -# --------------------------------------------------------------------------- -# Admonitions -# --------------------------------------------------------------------------- - -def _process_admonitions(soup: Soup) -> None: - """Convert admonition divs to blockquotes with a bold 'Type: Title' heading. - - Input:
-

Recommended versions

-

Body text...

-
- - Output:
-

Caution: Recommended versions

-

Body text...

-
- """ - for admonition in soup.find_all("div", class_="admonition"): - classes = admonition.get("class", []) - admonition_type = next((c for c in classes if c != "admonition"), None) - - title_elem = admonition.find("p", class_="admonition-title") - title_text = title_elem.get_text(strip=True) if title_elem else "" - - blockquote = soup.new_tag("blockquote") - - # Bold title paragraph - title_p = soup.new_tag("p") - strong = soup.new_tag("strong") - prefix = f"{admonition_type.capitalize()}: " if admonition_type else "" - strong.string = f"{prefix}{title_text}" - title_p.append(strong) - blockquote.append(title_p) - - # Body: all children except the title - for child in list(admonition.children): - if child == title_elem: - continue - blockquote.append(child.extract()) - - admonition.replace_with(blockquote) - - -def _process_release_notes_filters(soup: Soup) -> None: - """Remove interactive release-notes filter UI elements.""" - for container in soup.find_all("div", class_="release-notes-filters"): - container.decompose() - - -# --------------------------------------------------------------------------- -# Card macros -# --------------------------------------------------------------------------- - -def _process_cards(soup: Soup) -> None: - """Convert card macro HTML structures into markdown-friendly lists with links.""" - for cards_div in soup.find_all("div", class_=lambda c: c and c.startswith("cards ")): - card_wrappers = cards_div.find_all("div", class_="card-wrapper") - - if not card_wrappers: - continue - - ul = soup.new_tag("ul") - - for card_wrapper in card_wrappers: - link = card_wrapper.find("a", class_="card") - if not link: - continue - - href = link.get("href", "") - if href.startswith("//"): - href = "https:" + href - - title_elem = link.find("p", class_="title") - description_elem = link.find("p", class_="description") - - if not title_elem: - continue - - title = title_elem.get_text(strip=True) - description = description_elem.get_text(strip=True) if description_elem else "" - - li = soup.new_tag("li") - link_tag = soup.new_tag("a", href=href) - link_tag.string = title - li.append(link_tag) - - if description: - li.append(soup.new_string(": ")) - li.append(soup.new_string(description)) - - ul.append(li) - - cards_div.replace_with(ul) - - -# --------------------------------------------------------------------------- -# Markdown post-processing (applied by hooks.py to the generated Markdown) -# --------------------------------------------------------------------------- - -def editions_from_frontmatter(frontmatter: dict) -> list: - """Map ``edition``/``editions`` frontmatter values to display names.""" - - def _to_list(value): - if isinstance(value, list): - return value - if isinstance(value, str): - return value.split() - return [] - - all_editions = _to_list(frontmatter.get("edition")) + _to_list(frontmatter.get("editions") or []) - return [FRONTMATTER_EDITION_DISPLAY.get(e, e) for e in all_editions if e] - - -_MACRO_RE = re.compile(r"\[\[=\s*(\w+)\s*=\]\]") - - -def expand_macros(text: str, variables: dict) -> str: - """Expand simple ``[[= name =]]`` macro variables (mkdocs-macros syntax). - - Only plain scalar variables are substituted; unknown or complex macros are - left untouched so callers can detect and handle them. - """ - - def _substitute(match: re.Match) -> str: - value = variables.get(match.group(1)) - return str(value) if isinstance(value, (str, int, float)) else match.group(0) - - return _MACRO_RE.sub(_substitute, text) - - -def inject_page_metadata(content: str, description: str = "", editions: list = ()) -> str: - """Insert the page description and an 'Editions: X, Y' line after the first h1 heading.""" - metadata_lines = [] - if description: - metadata_lines += ["", description] - if editions: - metadata_lines += ["", "Editions: " + ", ".join(editions)] - if not metadata_lines: - return content - - lines = content.split("\n") - for i, line in enumerate(lines): - if line.startswith("# "): - lines[i + 1:i + 1] = metadata_lines - return "\n".join(lines) - - return "\n".join(metadata_lines).lstrip("\n") + "\n\n" + content - - -# ![alt](url) or ![alt](url "title") -_IMAGE_RE = re.compile(r'!\[([^\]]*)\]\(([^)\s]+)((?:\s+"[^"]*")?)\)') - - -def absolutize_image_urls(content: str, base_url: str, page_dir: str) -> str: - """Rewrite relative image URLs in Markdown to absolute ones. - - The llmstxt plugin makes link hrefs absolute but not image srcs, so the - ``![alt](src)`` references generated by markdownify keep their - page-relative paths, which break in ``llms-full.txt``. This mirrors the - URL logic the plugin applies to links. - - ``base_url`` is the site base URL (trailing slash), ``page_dir`` the - page's directory relative to the site root (e.g. ``"cdp/cdp"``). - """ - - def _absolutize(match: re.Match) -> str: - alt, url, title = match.groups() - if not url.startswith(("/", "#")) and not urlparse(url).scheme: - relative_base = urljoin(base_url, page_dir + "/") if page_dir else base_url - url = urljoin(relative_base, url) - return f"![{alt}]({url}{title})" - - return _IMAGE_RE.sub(_absolutize, content) - - -_ORDERED_MARKER_RE = re.compile(r"^(\s*)(\d+)\. (.*)$") -_FENCE_OPEN_RE = re.compile(r"^(\s*)(`{3,}|~{3,})") -# Width of an ordered-list marker ("1. ") — mdformat normalizes markers to -# "1. " and indents item content by exactly this much. -_MARKER_WIDTH = 3 - - -def renumber_ordered_lists(content: str) -> str: - """Replace repeated '1.' ordered-list markers with sequential numbers. - - The llmstxt plugin runs mdformat on the converted Markdown, which keeps - the first marker of each ordered list (the list's start number) and - rewrites all following markers to "1." (its default numbering style). - This restores sequential numbers while keeping track of nesting: - - - a stack of (marker indent, counter) pairs tracks open ordered lists; - the counter starts at the list's first marker number, so intentional - start values (
    , or literal "2\\." paragraphs in the - source) are preserved, - - blank lines and lines indented to an item's content column (marker - indent + 3) are item continuation and don't interrupt numbering, - - a less-indented line closes the lists it's not a continuation of, - - fenced code blocks are passed through untouched. - - Known limitations (acceptable for LLM-oriented output): markers of items - >= 10 are one character wider than the 3-space content indent, and lists - inside blockquotes keep their "1." markers. - """ - lines = content.split("\n") - result = [] - stack = [] # [marker_indent, counter] per open ordered list, outermost first - fence = None # (fence_char, fence_length) while inside a fenced code block - - def close_lists(indent: int) -> None: - # A line not indented to the content column of an open list closes it. - while stack and indent < stack[-1][0] + _MARKER_WIDTH: - stack.pop() - - for line in lines: - if fence is not None: - stripped = line.strip() - if stripped and set(stripped) == {fence[0]} and len(stripped) >= fence[1]: - fence = None - result.append(line) - continue - - fence_match = _FENCE_OPEN_RE.match(line) - if fence_match: - close_lists(len(fence_match.group(1))) - marker = fence_match.group(2) - fence = (marker[0], len(marker)) - result.append(line) - continue - - if not line.strip(): - # Blank lines separate loose-list items and item paragraphs; they - # never terminate a list on their own in mdformat output. - result.append(line) - continue - - marker_match = _ORDERED_MARKER_RE.match(line) - if marker_match: - indent = len(marker_match.group(1)) - while stack and stack[-1][0] > indent: - stack.pop() - if stack and stack[-1][0] == indent: - stack[-1][1] += 1 - else: - # A new list keeps its first marker number as the start. - stack.append([indent, int(marker_match.group(2))]) - result.append(f"{marker_match.group(1)}{stack[-1][1]}. {marker_match.group(3)}") - continue - - close_lists(len(line) - len(line.lstrip())) - result.append(line) - - return "\n".join(result) +__all__ = ["preprocess"] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000..0f32f11dfa --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,17 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "ibexa-llms-txt" +version = "0.1.0" +description = "Shared MkDocs hooks and llmstxt preprocessing logic for Ibexa documentation sites." +requires-python = ">=3.9" +dependencies = [ + "beautifulsoup4", + "PyYAML", +] + +[tool.setuptools] +package-dir = {"" = "tools"} +packages = ["llms_txt"] diff --git a/requirements.txt b/requirements.txt index eb13f097ff..8edaf77560 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,4 @@ mkdocs-redirects==1.2.2 mkdocs-autolinks-plugin==0.7.1 Jinja2==3.1.6 mkdocs-llmstxt +-e . diff --git a/tests/python/conftest.py b/tests/python/conftest.py index b9018a2f6b..dd499f159d 100644 --- a/tests/python/conftest.py +++ b/tests/python/conftest.py @@ -1,5 +1,7 @@ import sys from pathlib import Path -# Make repo-root modules (llmstxt_preprocess.py) importable. +# Make repo-root shim modules (llmstxt_preprocess.py, update_llmstxt_config.py) +# importable; the llms_txt package itself is available via the editable +# install (`-e .` in requirements-dev.txt). sys.path.insert(0, str(Path(__file__).resolve().parents[2])) diff --git a/tests/python/test_image_urls.py b/tests/python/test_image_urls.py index d81118c173..ad75a197b3 100644 --- a/tests/python/test_image_urls.py +++ b/tests/python/test_image_urls.py @@ -1,4 +1,4 @@ -from llmstxt_preprocess import absolutize_image_urls +from llms_txt.llmstxt_preprocess import absolutize_image_urls BASE = "https://doc.ibexa.co/en/latest/" diff --git a/tests/python/test_page_metadata.py b/tests/python/test_page_metadata.py index 5029316e42..5900b5c363 100644 --- a/tests/python/test_page_metadata.py +++ b/tests/python/test_page_metadata.py @@ -1,4 +1,4 @@ -from llmstxt_preprocess import ( +from llms_txt.llmstxt_preprocess import ( editions_from_frontmatter, expand_macros, inject_page_metadata, diff --git a/tests/python/test_preprocess.py b/tests/python/test_preprocess.py index d8ef5e96bf..6a02b9e846 100644 --- a/tests/python/test_preprocess.py +++ b/tests/python/test_preprocess.py @@ -2,8 +2,8 @@ from bs4 import BeautifulSoup from mkdocs_llmstxt._internal.plugin import _converter -import llmstxt_preprocess -from llmstxt_preprocess import renumber_ordered_lists +import llms_txt.llmstxt_preprocess as llmstxt_preprocess +from llms_txt.llmstxt_preprocess import renumber_ordered_lists def to_markdown(html: str) -> str: diff --git a/tests/python/test_renumber.py b/tests/python/test_renumber.py index 12e1a403fe..aab58ce8ab 100644 --- a/tests/python/test_renumber.py +++ b/tests/python/test_renumber.py @@ -1,4 +1,4 @@ -from llmstxt_preprocess import renumber_ordered_lists +from llms_txt.llmstxt_preprocess import renumber_ordered_lists def test_flat_list(): diff --git a/tools/llms_txt/__init__.py b/tools/llms_txt/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tools/llms_txt/hooks.py b/tools/llms_txt/hooks.py new file mode 100644 index 0000000000..b6b3edd1c9 --- /dev/null +++ b/tools/llms_txt/hooks.py @@ -0,0 +1,92 @@ +""" +MkDocs hooks for the llmstxt Markdown-conversion feature. + +- Keeps the llmstxt plugin's ``sections`` config in sync with the ``nav`` + defined in ``mkdocs.yml``. +- Post-processes the Markdown generated by the llmstxt plugin. + +All content transformations live in ``llmstxt_preprocess``; this module is +only MkDocs event glue. Fully generic — no documentation-project-specific +content — so it's shared across doc sites via this installable package. +""" + +from __future__ import annotations + +from pathlib import Path, PurePosixPath +from typing import TYPE_CHECKING + +from llms_txt.llmstxt_preprocess import ( + absolutize_image_urls, + editions_from_frontmatter, + expand_macros, + inject_page_metadata, + renumber_ordered_lists, +) +from llms_txt.update_llmstxt_config import convert_nav_to_llmstxt_sections + +if TYPE_CHECKING: + from mkdocs.config.defaults import MkDocsConfig + from mkdocs.structure.pages import Page + + +def on_config(config: "MkDocsConfig") -> None: + """Populate llmstxt sections from nav before the build starts. + + The llmstxt plugin reads ``config.sections`` in its ``on_files`` event, + which fires after ``on_config``, so injecting here is the right place. + """ + nav = config.get("nav") + if not nav: + return + + llmstxt = config["plugins"].get("llmstxt") + if llmstxt is None: + return + + docs_dir = Path(config["docs_dir"]) + llmstxt.config.sections = convert_nav_to_llmstxt_sections(nav, docs_dir) + + +def on_page_content(html: str, *, page: "Page", config: "MkDocsConfig", **kwargs) -> None: + """Reformat the Markdown content generated by the llmstxt plugin for this page. + + Hooks run after plugins for every event, so at this point the llmstxt + plugin has already generated Markdown and stored it in ``_md_pages``. + We reformat here so the plugin writes the corrected content in its own + ``on_post_build`` (which also runs before ours, for the same reason). + """ + llmstxt = config["plugins"].get("llmstxt") + if llmstxt is None: + return + + src_uri = page.file.src_uri + page_info = llmstxt._md_pages.get(src_uri) + if page_info is None: + return + + frontmatter = _read_frontmatter(page, config) + editions = editions_from_frontmatter(frontmatter) + description = expand_macros(str(frontmatter.get("description") or ""), config.get("extra") or {}) + if "[[=" in description: + # Unresolved macros must not leak into the output. + description = "" + content = inject_page_metadata(page_info.content, description, editions) + content = renumber_ordered_lists(content) + # Same base URL and page directory the plugin uses for making link hrefs absolute. + page_dir = PurePosixPath(page.file.dest_uri).parent.as_posix() + content = absolutize_image_urls(content, llmstxt._base_url, page_dir) + if content != page_info.content: + llmstxt._md_pages[src_uri] = page_info._replace(content=content) + + +def _read_frontmatter(page: "Page", config: "MkDocsConfig") -> dict: + """Read the page's YAML frontmatter from its source file.""" + src_path = Path(config["docs_dir"]) / page.file.src_path + try: + from mkdocs.utils import meta as mkdocs_meta + with open(src_path, encoding="utf-8") as f: + raw = f.read() + _, frontmatter = mkdocs_meta.get_data(raw) + return frontmatter + except Exception: + return {} diff --git a/tools/llms_txt/llmstxt_preprocess.py b/tools/llms_txt/llmstxt_preprocess.py new file mode 100644 index 0000000000..97d83e3711 --- /dev/null +++ b/tools/llms_txt/llmstxt_preprocess.py @@ -0,0 +1,550 @@ +""" +Markdown conversion logic for the llmstxt plugin. + +This module is the single place holding all content transformations used to +produce the Markdown version of the documentation: + +- ``preprocess(soup, output)`` — HTML-level transforms invoked by the + mkdocs-llmstxt plugin (configured in ``plugins.yml``) before the HTML is + converted to Markdown. +- Markdown post-processing helpers (``renumber_ordered_lists``, + ``inject_edition_badges``, ``editions_from_frontmatter``) applied by + ``hooks.py`` to the Markdown the plugin generated. + +This package (``llms_txt``) is installed as a dependency by other Ibexa doc +sites, which each keep a thin root-level ``llmstxt_preprocess.py`` shim +(loaded by the mkdocs-llmstxt plugin via ``spec_from_file_location``, since +its ``preprocess:`` config option requires a real file path) that re-exports +``preprocess`` from here. Keep this module stateless (constants and pure +functions only). +""" + +import html as html_module +import re +from urllib.parse import urljoin, urlparse + +from bs4 import BeautifulSoup as Soup, NavigableString + +PILL_CLASS_TO_EDITION = { + "pill--lts-update": "LTS Update", + "pill--experience": "Experience", + "pill--commerce": "Commerce", + "pill--headless": "Headless", + "pill--new-feature": "New feature", + "pill--first-release": "First release", +} + +FRONTMATTER_EDITION_DISPLAY = { + "lts-update": "LTS Update", + "experience": "Experience", + "commerce": "Commerce", + "headless": "Headless", +} + + +def preprocess(soup: Soup, output: str) -> None: + """ + Preprocess HTML to improve markdown conversion. + + Runs with autoclean disabled so we can control the order: + 1. Expand tabbed sets with labels before autoclean removes tabbed-labels. + 2. Run autoclean-equivalent cleanup. + 3. Replace inline edition badge spans with readable text. + 4. Remove release notes filter UI. + 5. Convert card macros to markdown lists. + + Note: frontmatter edition injection is handled in hooks.py on_page_content, + where page.file.src_path is available directly. + """ + _process_tabbed_sets(soup) + _autoclean(soup) + _process_inline_pills(soup) + _process_release_note_tags(soup) + _process_release_note_dates(soup) + _process_release_notes_filters(soup) + _process_cards(soup) + _process_info_tiles(soup) + _process_tables(soup) + _process_admonitions(soup) + + +# --------------------------------------------------------------------------- +# Tables +# --------------------------------------------------------------------------- + +def _process_tables(soup: Soup) -> None: + """Replace ✔ (U+2714) with 'Yes' in table cells for readability.""" + CHECK = "✔" + for cell in soup.find_all(["td", "th"]): + for text_node in cell.find_all(string=lambda s: CHECK in s): + text_node.replace_with(NavigableString(text_node.replace(CHECK, "Yes"))) + + +# --------------------------------------------------------------------------- +# Tabbed sets +# --------------------------------------------------------------------------- + +def _process_tabbed_sets(soup: Soup) -> None: + """Prepend each tab label as bold text before its content block.""" + for tabbed_set in soup.find_all("div", class_="tabbed-set"): + labels_div = tabbed_set.find("div", class_="tabbed-labels") + content_div = tabbed_set.find("div", class_="tabbed-content") + + if not labels_div or not content_div: + tabbed_set.unwrap() + continue + + labels = [label.get_text(strip=True) for label in labels_div.find_all("label")] + blocks = content_div.find_all("div", class_="tabbed-block", recursive=False) + + wrapper = soup.new_tag("div") + for i, block in enumerate(blocks): + if i < len(labels): + label_tag = soup.new_tag("p") + strong = soup.new_tag("strong") + strong.string = labels[i] + label_tag.append(strong) + wrapper.append(label_tag) + for child in list(block.children): + wrapper.append(child.extract()) + + tabbed_set.replace_with(wrapper) + + +# --------------------------------------------------------------------------- +# Autoclean equivalent (mirrors mkdocs-llmstxt autoclean, minus tabbed-labels) +# --------------------------------------------------------------------------- + +def _autoclean(soup: Soup) -> None: + """Replicate the plugin's autoclean so we can run it after tab processing.""" + + # Unwrap links that wrap an image (e.g. lightbox links) so the image + # itself decides its fate below. + for link in soup.find_all("a"): + img = link.find("img") + if img: + link.replace_with(img.extract()) + + # Keep images that have alt text so markdownify emits ![alt](src) + # (URLs are made absolute later by absolutize_image_urls); drop the rest, + # since an image without a description tells an AI agent nothing. + # Done in its own pass: mutating the tree from inside a find_all() predicate + # makes the traversal skip the elements that follow the mutated one. + for img in soup.find_all("img"): + alt = (img.get("alt") or "").strip() + if not alt: + img.decompose() + + def _should_remove(tag) -> bool: + if tag.name == "svg": + return True + classes = tag.get("class") or () + if tag.name == "a" and "headerlink" in classes: + return True + if "twemoji" in classes: + return True + # tabbed-labels are already consumed by _process_tabbed_sets, but + # handle any stragglers defensively. + if "tabbed-labels" in classes: + return True + return False + + for element in soup.find_all(_should_remove): + element.decompose() + + for element in soup.find_all("autoref"): + element.replace_with(NavigableString(element.get_text())) + + # Insert ", " between adjacent elements in table cells. + # html.parser silently drops
    (invalid void-closing tag), so adjacent + # blocks have no separator and markdownify concatenates their backticks. + for td in soup.find_all(["td", "th"]): + children = list(td.children) + for i in range(len(children) - 1): + curr = children[i] + nxt = children[i + 1] + if getattr(curr, "name", None) == "code" and getattr(nxt, "name", None) == "code": + curr.insert_after(NavigableString(", ")) + + for element in soup.find_all("div", attrs={"class": "doc-md-description"}): + element.replace_with(NavigableString(element.get_text().strip())) + + for element in soup.find_all("span", attrs={"class": "doc-labels"}): + element.decompose() + + # Flatten line-numbered code blocks to a plain
    , keeping the
    +    # language-* class (emitted thanks to pygments_lang_class) on the 
    ,
    +    # where markdownify's code_language_callback looks for it.
    +    for element in soup.find_all("table", attrs={"class": "highlighttable"}):
    +        code_elem = element.find("code")
    +        if code_elem:
    +            classes = code_elem.get("class") or ()
    +            language = next((c for c in classes if c.startswith("language-")), "")
    +            attr = f' class="{language}"' if language else ""
    +            element.replace_with(
    +                Soup(f"{html_module.escape(code_elem.get_text())}
    ", "html.parser") + ) + +# --------------------------------------------------------------------------- +# Inline edition badge spans (from snippet includes) +# --------------------------------------------------------------------------- + +def _pill_edition(node) -> str: + """Return the edition name of an inline pill span, or '' if not one.""" + if getattr(node, "name", None) != "span": + return "" + classes = node.get("class") or [] + if "pill--inline" not in classes: + return "" + for pill_cls, edition_name in PILL_CLASS_TO_EDITION.items(): + if pill_cls in classes: + return edition_name + return "" + + +def _process_inline_pills(soup: Soup) -> None: + """Replace inline edition pill spans with readable text. + + Consecutive pills (possibly separated by whitespace) are merged into a + single parenthetical, e.g. ' (Experience, Commerce)' instead of + ' (Experience) (Commerce)'. + """ + for span in soup.find_all("span", class_="pill--inline"): + if span.parent is None: # already consumed as part of a previous run + continue + edition = _pill_edition(span) + if not edition: + continue + + # Collect the run of pills that follow, skipping whitespace between them. + editions = [edition] + consumed = [] + node = span.next_sibling + pending_whitespace = [] + while node is not None: + if isinstance(node, NavigableString) and not node.strip(): + pending_whitespace.append(node) + node = node.next_sibling + continue + next_edition = _pill_edition(node) + if not next_edition: + break + editions.append(next_edition) + consumed += pending_whitespace + [node] + pending_whitespace = [] + node = node.next_sibling + + for extra_node in consumed: + extra_node.extract() + span.replace_with(soup.new_string(f" ({', '.join(editions)})")) + + +def _process_release_note_tags(soup: Soup) -> None: + """Append edition labels from release-note__tags divs to their preceding heading. + + Release notes use a
    block after each

    + containing empty
    elements rendered via CSS. + This converts them to a readable parenthetical on the heading, e.g.: + ## Google Gemini connector v5.0.7 (Headless, Experience, LTS Update, New feature) + """ + for tags_div in soup.find_all("div", class_="release-note__tags"): + editions = [] + for pill_div in tags_div.find_all("div"): + classes = pill_div.get("class", []) + for pill_cls, name in PILL_CLASS_TO_EDITION.items(): + if pill_cls in classes: + editions.append(name) + break + + heading = tags_div.find_previous_sibling(["h1", "h2", "h3", "h4"]) + if heading and editions: + # Insert before the permalink anchor so it's part of the heading text + anchor = heading.find("a", class_="headerlink") + label = NavigableString(f" ({', '.join(editions)})") + if anchor: + anchor.insert_before(label) + else: + heading.append(label) + + tags_div.decompose() + + +def _process_release_note_dates(soup: Soup) -> None: + """Prefix release-note dates with 'Release date: ' so the bare date line stays understandable.""" + for date_div in soup.find_all("div", class_="release-note__date"): + date_text = date_div.get_text(strip=True) + if date_text: + date_div.string = f"Release date: {date_text}" + + +def _process_info_tiles(soup: Soup) -> None: + """Simplify info-tile links to a single clean link text. + + Info tiles have a 'Details' label div and a separate content div. + After SVG removal, markdownify produces broken multi-line link text. + Replace the whole content with just the meaningful text. + """ + for tile in soup.find_all("a", class_="info-tile"): + # The label div ("Details") can be discarded + label = tile.find("div", class_="info-tile__details") + if label: + label.decompose() + + # Flatten remaining content to plain text + text = tile.get_text(separator=" ", strip=True) + tile.clear() + tile.append(soup.new_string(text)) + + +# --------------------------------------------------------------------------- +# Admonitions +# --------------------------------------------------------------------------- + +def _process_admonitions(soup: Soup) -> None: + """Convert admonition divs to blockquotes with a bold 'Type: Title' heading. + + Input:
    +

    Recommended versions

    +

    Body text...

    +
    + + Output:
    +

    Caution: Recommended versions

    +

    Body text...

    +
    + """ + for admonition in soup.find_all("div", class_="admonition"): + classes = admonition.get("class", []) + admonition_type = next((c for c in classes if c != "admonition"), None) + + title_elem = admonition.find("p", class_="admonition-title") + title_text = title_elem.get_text(strip=True) if title_elem else "" + + blockquote = soup.new_tag("blockquote") + + # Bold title paragraph + title_p = soup.new_tag("p") + strong = soup.new_tag("strong") + prefix = f"{admonition_type.capitalize()}: " if admonition_type else "" + strong.string = f"{prefix}{title_text}" + title_p.append(strong) + blockquote.append(title_p) + + # Body: all children except the title + for child in list(admonition.children): + if child == title_elem: + continue + blockquote.append(child.extract()) + + admonition.replace_with(blockquote) + + +def _process_release_notes_filters(soup: Soup) -> None: + """Remove interactive release-notes filter UI elements.""" + for container in soup.find_all("div", class_="release-notes-filters"): + container.decompose() + + +# --------------------------------------------------------------------------- +# Card macros +# --------------------------------------------------------------------------- + +def _process_cards(soup: Soup) -> None: + """Convert card macro HTML structures into markdown-friendly lists with links.""" + for cards_div in soup.find_all("div", class_=lambda c: c and c.startswith("cards ")): + card_wrappers = cards_div.find_all("div", class_="card-wrapper") + + if not card_wrappers: + continue + + ul = soup.new_tag("ul") + + for card_wrapper in card_wrappers: + link = card_wrapper.find("a", class_="card") + if not link: + continue + + href = link.get("href", "") + if href.startswith("//"): + href = "https:" + href + + title_elem = link.find("p", class_="title") + description_elem = link.find("p", class_="description") + + if not title_elem: + continue + + title = title_elem.get_text(strip=True) + description = description_elem.get_text(strip=True) if description_elem else "" + + li = soup.new_tag("li") + link_tag = soup.new_tag("a", href=href) + link_tag.string = title + li.append(link_tag) + + if description: + li.append(soup.new_string(": ")) + li.append(soup.new_string(description)) + + ul.append(li) + + cards_div.replace_with(ul) + + +# --------------------------------------------------------------------------- +# Markdown post-processing (applied by hooks.py to the generated Markdown) +# --------------------------------------------------------------------------- + +def editions_from_frontmatter(frontmatter: dict) -> list: + """Map ``edition``/``editions`` frontmatter values to display names.""" + + def _to_list(value): + if isinstance(value, list): + return value + if isinstance(value, str): + return value.split() + return [] + + all_editions = _to_list(frontmatter.get("edition")) + _to_list(frontmatter.get("editions") or []) + return [FRONTMATTER_EDITION_DISPLAY.get(e, e) for e in all_editions if e] + + +_MACRO_RE = re.compile(r"\[\[=\s*(\w+)\s*=\]\]") + + +def expand_macros(text: str, variables: dict) -> str: + """Expand simple ``[[= name =]]`` macro variables (mkdocs-macros syntax). + + Only plain scalar variables are substituted; unknown or complex macros are + left untouched so callers can detect and handle them. + """ + + def _substitute(match: re.Match) -> str: + value = variables.get(match.group(1)) + return str(value) if isinstance(value, (str, int, float)) else match.group(0) + + return _MACRO_RE.sub(_substitute, text) + + +def inject_page_metadata(content: str, description: str = "", editions: list = ()) -> str: + """Insert the page description and an 'Editions: X, Y' line after the first h1 heading.""" + metadata_lines = [] + if description: + metadata_lines += ["", description] + if editions: + metadata_lines += ["", "Editions: " + ", ".join(editions)] + if not metadata_lines: + return content + + lines = content.split("\n") + for i, line in enumerate(lines): + if line.startswith("# "): + lines[i + 1:i + 1] = metadata_lines + return "\n".join(lines) + + return "\n".join(metadata_lines).lstrip("\n") + "\n\n" + content + + +# ![alt](url) or ![alt](url "title") +_IMAGE_RE = re.compile(r'!\[([^\]]*)\]\(([^)\s]+)((?:\s+"[^"]*")?)\)') + + +def absolutize_image_urls(content: str, base_url: str, page_dir: str) -> str: + """Rewrite relative image URLs in Markdown to absolute ones. + + The llmstxt plugin makes link hrefs absolute but not image srcs, so the + ``![alt](src)`` references generated by markdownify keep their + page-relative paths, which break in ``llms-full.txt``. This mirrors the + URL logic the plugin applies to links. + + ``base_url`` is the site base URL (trailing slash), ``page_dir`` the + page's directory relative to the site root (e.g. ``"cdp/cdp"``). + """ + + def _absolutize(match: re.Match) -> str: + alt, url, title = match.groups() + if not url.startswith(("/", "#")) and not urlparse(url).scheme: + relative_base = urljoin(base_url, page_dir + "/") if page_dir else base_url + url = urljoin(relative_base, url) + return f"![{alt}]({url}{title})" + + return _IMAGE_RE.sub(_absolutize, content) + + +_ORDERED_MARKER_RE = re.compile(r"^(\s*)(\d+)\. (.*)$") +_FENCE_OPEN_RE = re.compile(r"^(\s*)(`{3,}|~{3,})") +# Width of an ordered-list marker ("1. ") — mdformat normalizes markers to +# "1. " and indents item content by exactly this much. +_MARKER_WIDTH = 3 + + +def renumber_ordered_lists(content: str) -> str: + """Replace repeated '1.' ordered-list markers with sequential numbers. + + The llmstxt plugin runs mdformat on the converted Markdown, which keeps + the first marker of each ordered list (the list's start number) and + rewrites all following markers to "1." (its default numbering style). + This restores sequential numbers while keeping track of nesting: + + - a stack of (marker indent, counter) pairs tracks open ordered lists; + the counter starts at the list's first marker number, so intentional + start values (
      , or literal "2\\." paragraphs in the + source) are preserved, + - blank lines and lines indented to an item's content column (marker + indent + 3) are item continuation and don't interrupt numbering, + - a less-indented line closes the lists it's not a continuation of, + - fenced code blocks are passed through untouched. + + Known limitations (acceptable for LLM-oriented output): markers of items + >= 10 are one character wider than the 3-space content indent, and lists + inside blockquotes keep their "1." markers. + """ + lines = content.split("\n") + result = [] + stack = [] # [marker_indent, counter] per open ordered list, outermost first + fence = None # (fence_char, fence_length) while inside a fenced code block + + def close_lists(indent: int) -> None: + # A line not indented to the content column of an open list closes it. + while stack and indent < stack[-1][0] + _MARKER_WIDTH: + stack.pop() + + for line in lines: + if fence is not None: + stripped = line.strip() + if stripped and set(stripped) == {fence[0]} and len(stripped) >= fence[1]: + fence = None + result.append(line) + continue + + fence_match = _FENCE_OPEN_RE.match(line) + if fence_match: + close_lists(len(fence_match.group(1))) + marker = fence_match.group(2) + fence = (marker[0], len(marker)) + result.append(line) + continue + + if not line.strip(): + # Blank lines separate loose-list items and item paragraphs; they + # never terminate a list on their own in mdformat output. + result.append(line) + continue + + marker_match = _ORDERED_MARKER_RE.match(line) + if marker_match: + indent = len(marker_match.group(1)) + while stack and stack[-1][0] > indent: + stack.pop() + if stack and stack[-1][0] == indent: + stack[-1][1] += 1 + else: + # A new list keeps its first marker number as the start. + stack.append([indent, int(marker_match.group(2))]) + result.append(f"{marker_match.group(1)}{stack[-1][1]}. {marker_match.group(3)}") + continue + + close_lists(len(line) - len(line.lstrip())) + result.append(line) + + return "\n".join(result) diff --git a/tools/llms_txt/update_llmstxt_config.py b/tools/llms_txt/update_llmstxt_config.py new file mode 100644 index 0000000000..b21d3ef261 --- /dev/null +++ b/tools/llms_txt/update_llmstxt_config.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +""" +Update the llmstxt plugin configuration in plugins.yml based on mkdocs.yml nav structure. +This script converts the mkdocs navigation into a format suitable for the llmstxt plugin. + +Files can be excluded from the llmstxt output by adding the following to their YAML frontmatter: + + exclude_from_llmstxt: true + +This excludes the page from both llms.txt and llms-full.txt, skips generating its Markdown +version, and hides the page action buttons. There is no per-file control for excluding a page +from llms.txt only (both files are derived from the same `sections` config). +Files without this property are included by default. +""" + +import re +import yaml +from pathlib import Path + + +def read_frontmatter(file_path): + """ + Parse YAML frontmatter from a markdown file. + Returns a dict of frontmatter values, or an empty dict if none is found. + Handles files that begin with HTML comments before the frontmatter block. + """ + try: + content = Path(file_path).read_text(encoding='utf-8') + except (OSError, UnicodeDecodeError): + return {} + + # Strip leading HTML comments (e.g. ) + content = re.sub(r'\A(\s*\s*)+', '', content, flags=re.DOTALL) + + if not content.startswith('---'): + return {} + + end = content.find('\n---', 3) + if end == -1: + return {} + + try: + return yaml.safe_load(content[3:end]) or {} + except yaml.YAMLError: + return {} + + +def is_excluded(file_path, docs_dir): + """ + Return True if the file has 'exclude_from_llmstxt: true' in its frontmatter. + file_path is relative to docs_dir (as written in mkdocs nav). + """ + full_path = Path(docs_dir) / file_path + fm = read_frontmatter(full_path) + return fm.get('exclude_from_llmstxt', False) is True + + +def convert_nav_to_llmstxt_sections(nav_list, docs_dir): + """ + Convert mkdocs nav list to llmstxt sections format. + Returns a dict mapping top-level section names to flat lists of file paths. + Files with 'exclude_from_llmstxt: true' in their frontmatter are skipped. + Sections that contain no remaining files are omitted. + """ + sections = {} + + def extract_files(item): + """Recursively extract included file paths from a nav item.""" + files = [] + if isinstance(item, str): + if not item.endswith('.html') and not is_excluded(item, docs_dir): + files.append(item) + elif isinstance(item, list): + for subitem in item: + files.extend(extract_files(subitem)) + elif isinstance(item, dict): + for value in item.values(): + if isinstance(value, str): + if not value.endswith('.html') and not is_excluded(value, docs_dir): + files.append(value) + elif isinstance(value, list): + for subitem in value: + files.extend(extract_files(subitem)) + return files + + for item in nav_list: + if isinstance(item, dict): + for section_name, section_content in item.items(): + files = extract_files({section_name: section_content}) + if files: + sections[section_name] = files + elif isinstance(item, str): + if not item.endswith('.html') and not is_excluded(item, docs_dir): + sections.setdefault('Ibexa Developer Documentation', []).append(item) + + return sections + + +def update_plugins_yml(plugins_path, mkdocs_path): + """ + Update the llmstxt plugin configuration in plugins.yml based on mkdocs.yml nav. + """ + with open(plugins_path, 'r') as f: + plugins_data = yaml.safe_load(f) + + with open(mkdocs_path, 'r') as f: + mkdocs_data = yaml.safe_load(f) + + # docs/ directory is resolved relative to mkdocs.yml location + docs_dir = Path(mkdocs_path).parent / mkdocs_data.get('docs_dir', 'docs') + + nav = mkdocs_data.get('nav', []) + new_sections = convert_nav_to_llmstxt_sections(nav, docs_dir) + + plugins_list = plugins_data.get('plugins', []) + for plugin in plugins_list: + if isinstance(plugin, dict) and 'llmstxt' in plugin: + plugin['llmstxt']['sections'] = new_sections + print(f"✓ Updated llmstxt plugin configuration") + print(f" Total sections: {len(new_sections)}") + break + else: + print("✗ llmstxt plugin not found in plugins.yml") + return False + + with open(plugins_path, 'w') as f: + yaml.dump(plugins_data, f, default_flow_style=False, sort_keys=False, + allow_unicode=True, width=120) + + print(f"✓ Updated {plugins_path}") + return True diff --git a/update_llmstxt_config.py b/update_llmstxt_config.py index b232392c38..f2ad8b04a0 100644 --- a/update_llmstxt_config.py +++ b/update_llmstxt_config.py @@ -1,135 +1,14 @@ #!/usr/bin/env python3 """ -Update the llmstxt plugin configuration in plugins.yml based on mkdocs.yml nav structure. -This script converts the mkdocs navigation into a format suitable for the llmstxt plugin. +CLI wrapper to update the llmstxt plugin configuration in plugins.yml based on mkdocs.yml nav structure. -Files can be excluded from the llmstxt output by adding the following to their YAML frontmatter: - - exclude_from_llmstxt: true - -This excludes the page from both llms.txt and llms-full.txt, skips generating its Markdown -version, and hides the page action buttons. There is no per-file control for excluding a page -from llms.txt only (both files are derived from the same `sections` config). -Files without this property are included by default. +The actual conversion logic lives in the installable ``llms_txt`` package; +this script just runs it against this repo's plugins.yml/mkdocs.yml. """ -import re -import yaml from pathlib import Path - -def read_frontmatter(file_path): - """ - Parse YAML frontmatter from a markdown file. - Returns a dict of frontmatter values, or an empty dict if none is found. - Handles files that begin with HTML comments before the frontmatter block. - """ - try: - content = Path(file_path).read_text(encoding='utf-8') - except (OSError, UnicodeDecodeError): - return {} - - # Strip leading HTML comments (e.g. ) - content = re.sub(r'\A(\s*\s*)+', '', content, flags=re.DOTALL) - - if not content.startswith('---'): - return {} - - end = content.find('\n---', 3) - if end == -1: - return {} - - try: - return yaml.safe_load(content[3:end]) or {} - except yaml.YAMLError: - return {} - - -def is_excluded(file_path, docs_dir): - """ - Return True if the file has 'exclude_from_llmstxt: true' in its frontmatter. - file_path is relative to docs_dir (as written in mkdocs nav). - """ - full_path = Path(docs_dir) / file_path - fm = read_frontmatter(full_path) - return fm.get('exclude_from_llmstxt', False) is True - - -def convert_nav_to_llmstxt_sections(nav_list, docs_dir): - """ - Convert mkdocs nav list to llmstxt sections format. - Returns a dict mapping top-level section names to flat lists of file paths. - Files with 'exclude_from_llmstxt: true' in their frontmatter are skipped. - Sections that contain no remaining files are omitted. - """ - sections = {} - - def extract_files(item): - """Recursively extract included file paths from a nav item.""" - files = [] - if isinstance(item, str): - if not item.endswith('.html') and not is_excluded(item, docs_dir): - files.append(item) - elif isinstance(item, list): - for subitem in item: - files.extend(extract_files(subitem)) - elif isinstance(item, dict): - for value in item.values(): - if isinstance(value, str): - if not value.endswith('.html') and not is_excluded(value, docs_dir): - files.append(value) - elif isinstance(value, list): - for subitem in value: - files.extend(extract_files(subitem)) - return files - - for item in nav_list: - if isinstance(item, dict): - for section_name, section_content in item.items(): - files = extract_files({section_name: section_content}) - if files: - sections[section_name] = files - elif isinstance(item, str): - if not item.endswith('.html') and not is_excluded(item, docs_dir): - sections.setdefault('Ibexa Developer Documentation', []).append(item) - - return sections - - -def update_plugins_yml(plugins_path, mkdocs_path): - """ - Update the llmstxt plugin configuration in plugins.yml based on mkdocs.yml nav. - """ - with open(plugins_path, 'r') as f: - plugins_data = yaml.safe_load(f) - - with open(mkdocs_path, 'r') as f: - mkdocs_data = yaml.safe_load(f) - - # docs/ directory is resolved relative to mkdocs.yml location - docs_dir = Path(mkdocs_path).parent / mkdocs_data.get('docs_dir', 'docs') - - nav = mkdocs_data.get('nav', []) - new_sections = convert_nav_to_llmstxt_sections(nav, docs_dir) - - plugins_list = plugins_data.get('plugins', []) - for plugin in plugins_list: - if isinstance(plugin, dict) and 'llmstxt' in plugin: - plugin['llmstxt']['sections'] = new_sections - print(f"✓ Updated llmstxt plugin configuration") - print(f" Total sections: {len(new_sections)}") - break - else: - print("✗ llmstxt plugin not found in plugins.yml") - return False - - with open(plugins_path, 'w') as f: - yaml.dump(plugins_data, f, default_flow_style=False, sort_keys=False, - allow_unicode=True, width=120) - - print(f"✓ Updated {plugins_path}") - return True - +from llms_txt.update_llmstxt_config import update_plugins_yml if __name__ == '__main__': script_dir = Path(__file__).parent