diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 12882cf..5ddd776 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,17 @@ jobs: - run: pip install -e '.[dev]' - run: pytest -q + wasm: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: dtolnay/rust-toolchain@stable + with: + targets: wasm32-unknown-unknown + - run: cargo install wasm-bindgen-cli --version 0.2.128 + - run: npm run build + working-directory: wasm + build: needs: test strategy: @@ -80,6 +91,6 @@ jobs: - uses: dtolnay/rust-toolchain@stable - uses: rust-lang/crates-io-auth-action@v1 id: auth - - run: cargo publish + - run: cargo publish -p mdhtml-crate env: CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }} diff --git a/.gitignore b/.gitignore index 5c5458b..b9622b1 100644 --- a/.gitignore +++ b/.gitignore @@ -90,6 +90,11 @@ docs/_build/ # PyBuilder target/ +# npm +node_modules/ +wasm/pkg/ +wasm/package-lock.json + # Jupyter Notebook .ipynb_checkpoints diff --git a/Cargo.toml b/Cargo.toml index ef0b6f8..d73eca8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,11 +1,24 @@ -[package] -name = "mdhtml-crate" +[workspace] +members = ["py", "wasm"] + +[workspace.package] version = "0.1.39" edition = "2024" rust-version = "1.91" license = "Apache-2.0" -description = "A bounded-time, Pandoc-leaning Markdown parser with GFM, Extra/kramdown, math, fenced divs, and MDHTML output." repository = "https://github.com/AnswerDotAI/mdhtml" + +[workspace.lints.clippy] +too_many_arguments = "allow" + +[package] +name = "mdhtml-crate" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +description = "A bounded-time, Pandoc-leaning Markdown parser with GFM, Extra/kramdown, math, fenced divs, and MDHTML output." +repository.workspace = true homepage = "https://github.com/AnswerDotAI/mdhtml" documentation = "https://github.com/AnswerDotAI/mdhtml" keywords = ["markdown", "parser", "html", "commonmark", "gfm"] @@ -14,7 +27,6 @@ categories = ["parser-implementations", "text-processing"] [lib] name = "mdhtml" path = "src/lib.rs" -crate-type = ["cdylib", "rlib"] [profile.release] lto = false @@ -23,6 +35,9 @@ codegen-units = 16 [profile.release.package.mdhtml-crate] incremental = true +[profile.release.package.mdhtml-py] +incremental = true + [profile.dist] inherits = "release" lto = true @@ -33,16 +48,18 @@ strip = true [profile.dist.package.mdhtml-crate] incremental = false +[profile.dist.package.mdhtml-py] +incremental = false + +[profile.wasm] +inherits = "dist" +opt-level = "z" + [dependencies] base64 = "0.23.0" fast5ever = { git = "https://github.com/AnswerDotAI/fast5ever", version = "0.1.5" } html-escape = ">=0.2" -pyo3 = { version = ">=0.28", optional = true } unicode-properties = "0.1" -[features] -python = ["dep:pyo3"] -extension-module = ["python", "pyo3/extension-module"] - -[lints.clippy] -too_many_arguments = "allow" +[lints] +workspace = true diff --git a/DEV.md b/DEV.md index a706c50..1b67bd2 100644 --- a/DEV.md +++ b/DEV.md @@ -23,14 +23,34 @@ The `release` profile is optimized and incremental for fast local iteration. CI ```bash cargo fmt -cargo check -cargo test +cargo check --workspace +cargo clippy --workspace --all-targets +cargo test --workspace pytest -q chkstyle python/mdhtml tests ``` The Python tests in `tests/` exercise the built native extension and the fast5ever boundary. Rust integration tests also verify structured diagnostics and parse → canonical Markdown → parse preservation at the rendered MDHTML-tree boundary. +## Layout + +The repository is a Cargo workspace with one published crate and one binding crate per consumer. `src/` is the `mdhtml-crate` library: the parser, the `Document` model, the renderers, and the exporters, with no knowledge of any host language. The binding crates are never published to crates.io. The version lives once, in `[workspace.package]` of the root `Cargo.toml`, and every member inherits it. + +`py/` is `mdhtml-py`, the PyO3 glue that `python/mdhtml/` imports as `mdhtml._native`. maturin builds it through `manifest-path` in `pyproject.toml`. + +`wasm/` is `mdhtml-wasm`, the `wasm-bindgen` glue for the browser, and `wasm/package.json` is the npm package `@answerdotai/mdhtml` around it. The package is private until its first publish. Its `version` field is a copy that `ship-bump` keeps in step through `[tool.fastship].version-files`. + +Building needs two one-time installs. The CLI version must match the crate version pinned in `wasm/Cargo.toml`. + +```bash +rustup target add wasm32-unknown-unknown +cargo install wasm-bindgen-cli --version 0.2.128 +``` + +`npm run build` in `wasm/` compiles with the `wasm` profile (`dist` at `opt-level = "z"`, for size) and runs `wasm-bindgen` into the ignored `wasm/pkg/`: the `.wasm`, the JavaScript glue, and type declarations. That is the `maturin develop` of the JavaScript side. In the browser the output of `md2mdhtml` goes straight into the DOM, and the browser's own parser does the tree construction that fast5ever does for Python. + +A binding crate can only reach the library's public surface, so anything a binding needs is exported from `src/lib.rs`. The Python glue needs six items beyond the documented API (`render_inlines`, `plain`, `code_block_open`, `CODE_BLOCK_CLOSE`, `trailing_attr_span`, `highlight_md`), exported by name so the modules that hold them stay private. + ## Shared MDHTML core MDHTML is the normative cross-format IR. `Document` is its typed Rust construction model; attributes, structured diagnostics, UTF-8-safe lines, bounded scans, and semantic serializers live in this crate so additional source importers can reuse them directly. Source-specific syntax structures remain private and transient. @@ -90,7 +110,7 @@ Release flow is: release first, then bump. pytest -q ``` -2. Confirm the release version in `Cargo.toml` (`[package].version`). `pyproject.toml` gets the Python package version from Cargo via `dynamic = ["version"]`. +2. Confirm the release version in `Cargo.toml` (`[workspace.package].version`; every crate inherits it). `pyproject.toml` gets the Python package version from Cargo via `dynamic = ["version"]`. 3. Release: diff --git a/py/Cargo.toml b/py/Cargo.toml new file mode 100644 index 0000000..baa1d8e --- /dev/null +++ b/py/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "mdhtml-py" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +publish = false + +[lints] +workspace = true + +[lib] +name = "mdhtml_native" +crate-type = ["cdylib"] + +[dependencies] +mdhtml-crate = { path = ".." } +pyo3 = ">=0.28" + +[features] +extension-module = ["pyo3/extension-module"] diff --git a/src/python.rs b/py/src/lib.rs similarity index 95% rename from src/python.rs rename to py/src/lib.rs index 1768d91..0a02713 100644 --- a/src/python.rs +++ b/py/src/lib.rs @@ -6,11 +6,11 @@ use pyo3::pybacked::PyBackedStr; use pyo3::types::{PyDict, PyTuple}; use std::collections::{HashMap, HashSet}; -use crate::ast::{Attr, Block, Document, Inline}; -use crate::inline::EditNode; -use crate::render::{CODE_BLOCK_CLOSE, code_block_open, plain, render_document, render_inlines}; -use crate::resolve; -use crate::{MathMode, Options, TemplateDelimiter, TemplateForm}; +use mdhtml::EditNode; +use mdhtml::ast::{Attr, Block, Document, Inline}; +use mdhtml::resolve; +use mdhtml::{CODE_BLOCK_CLOSE, code_block_open, plain, render as render_document, render_inlines}; +use mdhtml::{MathMode, Options, TemplateDelimiter, TemplateForm}; type TemplateArg = (String, String, String, Option<(String, String)>, String, Option<(String, String, String)>); @@ -46,7 +46,7 @@ fn md2mdhtml( Options { math: parse_math_mode(math)?, bare_autolinks, implicit_figures, templates: parse_templates(templates)?, frontmatter, ..Options::default() }; if let Some(depth) = max_block_depth { options.max_block_depth = depth; } if let Some(depth) = max_link_paren_depth { options.max_link_paren_depth = depth; } - let mut doc = guard("parsing markdown", || py.detach(|| crate::parse(markdown, &options)))?; + let mut doc = guard("parsing markdown", || py.detach(|| mdhtml::parse(markdown, &options)))?; if let Some(callbacks) = callbacks { apply_callbacks(&mut doc, &callbacks)? } let warnings = std::mem::take(&mut doc.diagnostics).into_iter().map(|diagnostic| diagnostic.to_string()).collect(); let meta = std::mem::take(&mut doc.meta); @@ -56,20 +56,20 @@ fn md2mdhtml( #[pyfunction] /// Render canonical MDHTML as deterministic `md` dialect source. -fn mdhtml2md(py: Python<'_>, mdhtml: &str) -> PyResult { guard("rendering MDHTML as Markdown", || py.detach(|| crate::mdhtml2md(mdhtml))) } +fn mdhtml2md(py: Python<'_>, mdhtml: &str) -> PyResult { guard("rendering MDHTML as Markdown", || py.detach(|| mdhtml::mdhtml2md(mdhtml))) } #[pyfunction] #[pyo3(signature = (markdown, width=None))] fn wrap_md(py: Python<'_>, markdown: &str, width: Option) -> PyResult { if width == Some(0) { return Err(PyValueError::new_err("width must be positive")); } - guard("wrapping Markdown", || py.detach(|| crate::wrap_md(markdown, width))) + guard("wrapping Markdown", || py.detach(|| mdhtml::wrap_md(markdown, width))) } #[pyfunction] #[pyo3(signature = (markdown, target_words=700))] fn md_chunks(py: Python<'_>, markdown: &str, target_words: usize) -> PyResult> { if target_words == 0 { return Err(PyValueError::new_err("target_words must be positive")); } - guard("chunking Markdown", || py.detach(|| crate::md_chunks(markdown, target_words))) + guard("chunking Markdown", || py.detach(|| mdhtml::md_chunks(markdown, target_words))) .map(|chunks| chunks.into_iter().map(|chunk| (chunk.md, chunk.start.as_str())).collect()) } @@ -77,7 +77,7 @@ fn md_chunks(py: Python<'_>, markdown: &str, target_words: usize) -> PyResult, markdown: &str, target_words: usize, length_scale: usize) -> PyResult> { if target_words == 0 || length_scale == 0 { return Err(PyValueError::new_err("target_words and length_scale must be positive")); } - guard("chunking Markdown", || py.detach(|| crate::md_chunks_greedy(markdown, target_words, length_scale))) + guard("chunking Markdown", || py.detach(|| mdhtml::md_chunks_greedy(markdown, target_words, length_scale))) .map(|chunks| chunks.into_iter().map(|chunk| (chunk.md, chunk.start.as_str())).collect()) } @@ -85,7 +85,7 @@ fn md_chunks_greedy(py: Python<'_>, markdown: &str, target_words: usize, length_ #[pyo3(signature = (markdown, target_words=700))] fn md_chunks_structural(py: Python<'_>, markdown: &str, target_words: usize) -> PyResult> { if target_words == 0 { return Err(PyValueError::new_err("target_words must be positive")); } - guard("chunking Markdown", || py.detach(|| crate::md_chunks_structural(markdown, target_words))) + guard("chunking Markdown", || py.detach(|| mdhtml::md_chunks_structural(markdown, target_words))) .map(|chunks| chunks.into_iter().map(|chunk| (chunk.md, chunk.start.as_str())).collect()) } @@ -97,7 +97,7 @@ fn md_chunks_structural_batch(py: Python<'_>, markdown: Vec, target py.detach(move || { markdown .into_iter() - .map(|source| crate::md_chunks_structural(&source, target_words).into_iter().map(|chunk| (chunk.md, chunk.start.as_str())).collect()) + .map(|source| mdhtml::md_chunks_structural(&source, target_words).into_iter().map(|chunk| (chunk.md, chunk.start.as_str())).collect()) .collect() }) }) @@ -105,7 +105,7 @@ fn md_chunks_structural_batch(py: Python<'_>, markdown: Vec, target #[pyfunction] fn wiki2mdhtml(py: Python<'_>, wikitext: &str) -> PyResult<(String, Vec)> { - let mut doc = guard("parsing wikitext", || py.detach(|| crate::parse_wikitext(wikitext)))?; + let mut doc = guard("parsing wikitext", || py.detach(|| mdhtml::parse_wikitext(wikitext)))?; let warnings = std::mem::take(&mut doc.diagnostics).into_iter().map(|diagnostic| diagnostic.to_string()).collect(); Ok((guard("rendering wikitext", || py.detach(|| render_document(&doc)))?, warnings)) } @@ -123,7 +123,7 @@ fn guard(what: &str, f: impl FnOnce() -> T) -> PyResult { fn blocks(py: Python<'_>, markdown: &str, math: &str, implicit_figures: bool, templates: Option>, nested: bool) -> PyResult>> { let options = Options { math: parse_math_mode(math)?, implicit_figures, nested_spans: nested, templates: parse_templates(templates)?, ..Options::default() }; - let spans = guard("parsing markdown", || py.detach(|| crate::block_spans(markdown, &options)))?; + let spans = guard("parsing markdown", || py.detach(|| mdhtml::block_spans(markdown, &options)))?; spans .into_iter() .map(|span| { @@ -158,7 +158,7 @@ fn blocks(py: Python<'_>, markdown: &str, math: &str, implicit_figures: bool, te #[pyo3(signature = (markdown, *, math = "brackets", templates = None))] fn anchors(py: Python<'_>, markdown: &str, math: &str, templates: Option>) -> PyResult>> { let options = Options { math: parse_math_mode(math)?, templates: parse_templates(templates)?, ..Options::default() }; - let doc = guard("parsing markdown", || crate::parse(markdown, &options))?; + let doc = guard("parsing markdown", || mdhtml::parse(markdown, &options))?; resolve::doc_anchors(&doc) .into_iter() .map(|(id, kind, text)| { @@ -177,12 +177,12 @@ fn target_kind(name: &str) -> Option<&'static str> { resolve::target_kind(name) /// Byte range of the trailing attribute group of `line`, or None: the group /// `strip_trailing_attr` would consume, as the dialect's own grammar judges it. #[pyfunction] -fn trailing_attr_span(line: &str) -> Option<(usize, usize)> { crate::attrs::trailing_attr_span(line) } +fn trailing_attr_span(line: &str) -> Option<(usize, usize)> { mdhtml::trailing_attr_span(line) } #[pyfunction] #[pyo3(signature = (markdown, *, math = "brackets", templates = None))] fn edit_nodes(py: Python<'_>, markdown: &str, math: &str, templates: Option>) -> PyResult>> { let options = Options { math: parse_math_mode(math)?, templates: parse_templates(templates)?, ..Options::default() }; - let nodes = guard("parsing markdown edit nodes", || crate::block::parse_edit_nodes(markdown, &options))?; + let nodes = guard("parsing markdown edit nodes", || mdhtml::edit_nodes(markdown, &options))?; nodes .into_iter() .map(|node| { @@ -309,7 +309,7 @@ fn dialect_css(preview: bool) -> String { resolve::dialect_css(preview) } /// scopes documented in `highlight.rs`. #[pyfunction] #[pyo3(signature = (src, class_prefix="hl-"))] -fn highlight_md(src: &str, class_prefix: &str) -> String { crate::highlight::highlight_md(src, class_prefix) } +fn highlight_md(src: &str, class_prefix: &str) -> String { mdhtml::highlight_md(src, class_prefix) } /// Heading numbering per a `{lvlText: numFmt}` scheme (Word semantics): /// `bump` at each heading, then read its display or full-context number. @@ -901,7 +901,7 @@ fn export_html( auto_ids: bool, gh_ids: bool, ) -> PyResult<(String, Vec)> { - use crate::export_html::{HlMode, HtmlExportOptions, NumberHeadings, RefsMode}; + use mdhtml::export_html::{HlMode, HtmlExportOptions, NumberHeadings, RefsMode}; let number_headings = match number_headings { None => None, Some(o) if o.is_none() => None, @@ -946,7 +946,7 @@ fn export_html( auto_ids, gh_ids, }; - let result = if hl_lang.is_none() && code_wrap.is_none() && hl_fn.is_none() { py.detach(|| crate::export_html::export_html(src, &opts)) } else { crate::export_html::export_html(src, &opts) }; + let result = if hl_lang.is_none() && code_wrap.is_none() && hl_fn.is_none() { py.detach(|| mdhtml::export_html::export_html(src, &opts)) } else { mdhtml::export_html::export_html(src, &opts) }; if let Some(e) = hook_err.into_inner().unwrap() { return Err(e); } result.map_err(vr) } diff --git a/pyproject.toml b/pyproject.toml index c9cd06a..c79a3e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,12 +31,13 @@ Issues = "https://github.com/AnswerDotAI/mdhtml/issues" [project.optional-dependencies] fill = ["execnb>=0.3.2"] hl = ["fastpylight>=0.1.18"] -dev = ["fastship>=0.0.14", "maturin~=1.0", "pytest", "execnb>=0.3.2", "fastpylight>=0.1.18", "math-core~=0.7.0"] +dev = ["fastship>=0.1.5", "maturin~=1.0", "pytest", "execnb>=0.3.2", "fastpylight>=0.1.18", "math-core~=0.7.0"] [project.entry-points.fastaudit_safe_native] mdhtml = "mdhtml" [tool.maturin] +manifest-path = "py/Cargo.toml" features = ["extension-module"] python-source = "python" module-name = "mdhtml._native" @@ -48,7 +49,8 @@ addopts = "-m 'not slow'" timeout = 60 [tool.uv] -cache-keys = [{ file = "pyproject.toml" }, { file = "src/**/*.rs" }, { file = "src/**/*.css" }, { file = "Cargo.toml" }, { file = ".git/fastws-cargo-key" }] +cache-keys = [{ file = "pyproject.toml" }, { file = "src/**/*.rs" }, { file = "src/**/*.css" }, { file = "py/**/*.rs" }, { file = "Cargo.toml" }, { file = "py/Cargo.toml" }, { file = ".git/fastws-cargo-key" }] [tool.fastship] branch = "main" +version-files = ["wasm/package.json"] diff --git a/src/lib.rs b/src/lib.rs index 09f7131..41e5165 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,33 +16,34 @@ mod highlight; mod inline; mod line; pub mod markdown; -#[cfg(feature = "python")] -mod python; mod render; pub mod resolve; pub mod scan; pub mod template; pub mod wikitext; -mod write_md; mod wrap; +mod write_md; pub use ast::{ Align, Attr, Block, DefinitionItem, DefinitionTerm, Document, Footnote, HtmlToken, Inline, ListItem, Operation, OperationArg, TableCell, TableCellData, TableRow, TableRowData, }; +pub use attrs::trailing_attr_span; pub use block::BlockSpan; pub use chunk::{ ChunkStart, MdChunk, MdChunkRange, document_chunk_ranges_structural, document_chunks_structural, md_chunks, md_chunks_greedy, md_chunks_structural, }; pub use diagnostic::{Diagnostic, Severity}; pub use fast5ever; +pub use highlight::highlight_md; pub use inline::{EditNode, XrefSeg}; pub use line::{LineOffset, SourceLocation, SourceSpan}; pub use markdown::{dom2md, mdhtml2md}; +pub use render::{CODE_BLOCK_CLOSE, code_block_open, plain, render_inlines}; pub use template::TokenKind; pub use wikitext::{parse as parse_wikitext, wiki2md, wiki2mdhtml}; -pub use write_md::render_md; pub use wrap::wrap_md; +pub use write_md::render_md; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum MathMode { diff --git a/src/render.rs b/src/render.rs index 8f521b1..8448b7f 100644 --- a/src/render.rs +++ b/src/render.rs @@ -10,8 +10,7 @@ pub(crate) fn render_document(doc: &Document) -> String { out } -#[allow(dead_code)] -pub(crate) fn render_inlines(items: &[Inline]) -> String { +pub fn render_inlines(items: &[Inline]) -> String { let doc = Document::default(); let mut r = Renderer::new(&doc); let mut out = String::new(); @@ -550,7 +549,7 @@ pub fn code_block_open(attr: &Attr, lang: Option<&str>) -> String { out } -pub(crate) fn plain(items: &[Inline]) -> String { +pub fn plain(items: &[Inline]) -> String { let mut out = String::new(); for item in items { match item { diff --git a/wasm/Cargo.toml b/wasm/Cargo.toml new file mode 100644 index 0000000..3fb48bd --- /dev/null +++ b/wasm/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "mdhtml-wasm" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +publish = false + +[lints] +workspace = true + +[lib] +crate-type = ["cdylib"] + +[dependencies] +mdhtml-crate = { path = ".." } +wasm-bindgen = "=0.2.128" diff --git a/wasm/package.json b/wasm/package.json new file mode 100644 index 0000000..ab35009 --- /dev/null +++ b/wasm/package.json @@ -0,0 +1,15 @@ +{ + "name": "@answerdotai/mdhtml", + "version": "0.1.39", + "description": "mdhtml's Markdown parser for the browser, compiled to WebAssembly", + "license": "Apache-2.0", + "repository": "github:AnswerDotAI/mdhtml", + "private": true, + "type": "module", + "main": "pkg/mdhtml_wasm.js", + "types": "pkg/mdhtml_wasm.d.ts", + "files": ["pkg"], + "scripts": { + "build": "cargo build -p mdhtml-wasm --target wasm32-unknown-unknown --profile wasm && wasm-bindgen --target web --out-dir pkg ../target/wasm32-unknown-unknown/wasm/mdhtml_wasm.wasm" + } +} diff --git a/wasm/src/lib.rs b/wasm/src/lib.rs new file mode 100644 index 0000000..0b3be5d --- /dev/null +++ b/wasm/src/lib.rs @@ -0,0 +1,12 @@ +//! JavaScript bindings for `mdhtml`. Each function here is one the browser +//! may call. `wasm-bindgen` generates the string marshalling around it. + +use mdhtml::{Options, parse, render}; +use wasm_bindgen::prelude::*; + +/// Render Markdown to an MDHTML fragment with the default options. The browser +/// parses the fragment, which is the tree-construction step fast5ever does in Python. +/// Python's `md2mdhtml` also returns warnings and frontmatter meta. This entry +/// returns the fragment alone until the JavaScript API needs them. +#[wasm_bindgen] +pub fn md2mdhtml(src: &str) -> String { render(&parse(src, &Options::default())) }