diff --git a/CHANGELOG.md b/CHANGELOG.md index 87fe4b20c..c00ae2f64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ Full release notes with details on each version: [GitHub Releases](https://githu ## 0.9.18 (2026-07-17) +- Feature: `graphify export obsidian --with-sources` wires each node note to its source content instead of leaving it a bare stub (#1968, thanks @SinghAman21). Every distinct `source_file` is copied into the vault under `sources/` as a real note (`_src_.md`, collision-safe) — markdown/text verbatim, other sources embedded in a syntax-highlighted fence — and each node note gains a `> [!info] Source` callout wikilinking to it, displayed with the original path. Paths are resolved relative to the scan root (the `#1941` layout) with the out dir as a fallback for pre-0.9.17 graphs; `--source-root PATH` overrides where they're resolved. The source notes are graphify-owned, so a re-export without the flag prunes them and the vault returns to plain notes; a node whose source can't be resolved on disk (or is over a 2 MB cap) simply gets no callout rather than a dangling link. - Fix: an incomplete extraction no longer force-writes a partial graph over a complete one (#1951, thanks @TPAteeq). A crashed AST/semantic pass, a some-chunks-failed run, or a walk that couldn't fully enumerate the corpus (permission-denied subtree) produced a smaller graph that the `to_json(force=True)` path wrote anyway, bypassing the #479 shrink guard; the `--no-cluster` raw dump had no guard at all. Both paths now refuse to overwrite a larger existing graph when the run was incomplete (exit 1, nothing written) unless `--allow-partial` is passed, and a present-but-unparseable existing graph fails closed (a corrupt/mid-write file could be hiding a complete graph). `detect()`'s `walk_errors` now count as incomplete too. - Fix: `graph.json`, `manifest.json`, and the other JSON artifacts are now written atomically (#1952, thanks @TPAteeq). A kill, OOM, or ENOSPC mid-write used to leave a truncated file — and a truncated `graph.json` then wedged every later run via the shrink guard's fail-safe. Writes now go to a temp file in the same directory and `os.replace` into place (writing *through* a symlink so shared-store setups keep working); the writers the original change missed (the `--no-cluster` dump, `merge-graphs`/`merge-chunks`/`merge-semantic`, the analysis/labels sidecars, and the global graph/manifest) are routed through it too. - Fix: truncated LLM chunks are no longer promoted to the semantic cache as complete (#1950, thanks @TPAteeq). A chunk that hit the output-token limit and couldn't be split further was cached and manifest-stamped as authoritative, so its incomplete node set replayed forever. Such chunks are now marked partial (treated as a cache miss and re-dispatched), including the common case where the truncation parses to *nothing*: the give-up sites record the chunk's own files independently of parsed items, so a sliced document whose later slice truncated empty is no longer stamped complete, and a clean slice can't re-promote a partial entry. diff --git a/graphify/cli.py b/graphify/cli.py index 27b3fb234..6f49e261b 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -1738,7 +1738,7 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph": print(" html [--graph PATH] [--labels PATH] [--node-limit N] [--no-viz]", file=sys.stderr) print(" callflow-html [GRAPH|DIR] [--graph PATH] [--labels PATH] [--report PATH] [--sections PATH] [--output HTML]", file=sys.stderr) print(" [--lang auto|zh-CN|en] [--max-sections N] [--diagram-scale N]", file=sys.stderr) - print(" obsidian [--graph PATH] [--labels PATH] [--dir PATH]", file=sys.stderr) + print(" obsidian [--graph PATH] [--labels PATH] [--dir PATH] [--with-sources] [--source-root PATH]", file=sys.stderr) print(" wiki [--graph PATH] [--labels PATH]", file=sys.stderr) print(" svg [--graph PATH] [--labels PATH]", file=sys.stderr) print(" graphml [--graph PATH]", file=sys.stderr) @@ -1767,6 +1767,11 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph": node_limit = 5000 no_viz = False obsidian_dir = Path(_GRAPHIFY_OUT) / "obsidian" + # --with-sources (#1968): copy each node's source file into the vault and + # wire a callout to it. --source-root overrides where the (scan-root- + # relative) source_file paths are resolved on disk. + obsidian_with_sources = False + obsidian_source_root: Path | None = None # Shared push-connection settings for the graph-database sinks (neo4j, # falkordb), parsed from the generic --push/--user/--password flags below. push_uri: str | None = None @@ -1828,6 +1833,10 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph": no_viz = True; i += 1 elif a == "--dir" and i + 1 < len(args): obsidian_dir = Path(args[i + 1]); i += 2 + elif a == "--with-sources": + obsidian_with_sources = True; i += 1 + elif a == "--source-root" and i + 1 < len(args): + obsidian_source_root = Path(args[i + 1]).expanduser(); i += 2 elif a == "--push" and i + 1 < len(args): push_uri = args[i + 1]; i += 2 elif a == "--user" and i + 1 < len(args): @@ -1973,9 +1982,22 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph": elif subcmd == "obsidian": from graphify.export import to_obsidian as _to_obsidian, to_canvas as _to_canvas + # source_file is stored relative to the scan root (#1941); resolve it + # against an explicit --source-root, else the graph's scan root + # (out_dir's parent under the default /graphify-out/ layout) with + # the out_dir itself as a fallback for graphs written by <=0.9.16 that + # stored source_file relative to the out dir (#1968). + if obsidian_with_sources: + _src_roots = ([obsidian_source_root] if obsidian_source_root + else [out_dir.parent, out_dir]) + else: + _src_roots = None n = _to_obsidian(G, communities, str(obsidian_dir), - community_labels=labels or None, cohesion=cohesion or None) + community_labels=labels or None, cohesion=cohesion or None, + with_sources=obsidian_with_sources, source_root=_src_roots) print(f"Obsidian vault: {n} notes in {obsidian_dir}/") + if obsidian_with_sources: + print(f" --with-sources: source notes in {obsidian_dir}/sources/") _to_canvas(G, communities, str(obsidian_dir / "graph.canvas"), community_labels=labels or None) print(f"Canvas: {obsidian_dir}/graph.canvas") diff --git a/graphify/export.py b/graphify/export.py index e1f2caa99..2e7be347c 100644 --- a/graphify/export.py +++ b/graphify/export.py @@ -451,12 +451,123 @@ def _dedup_node_filenames(G: nx.Graph, safe_name) -> dict[str, str]: return node_filenames +# --with-sources (#1968): map a source file's extension to a fenced-code +# language hint so non-markdown sources embed with syntax highlighting. +_SOURCE_FENCE_LANG = { + "py": "python", "js": "javascript", "jsx": "javascript", "ts": "typescript", + "tsx": "typescript", "rs": "rust", "go": "go", "java": "java", "rb": "ruby", + "php": "php", "c": "c", "h": "c", "cpp": "cpp", "hpp": "cpp", "cc": "cpp", + "cs": "csharp", "swift": "swift", "kt": "kotlin", "scala": "scala", + "sh": "bash", "bash": "bash", "zsh": "bash", "sql": "sql", "yaml": "yaml", + "yml": "yaml", "toml": "toml", "json": "json", "html": "html", "css": "css", + "xml": "xml", "lua": "lua", "r": "r", "jl": "julia", +} + +# Extensions Obsidian renders as prose, so their source note copies the file +# verbatim rather than wrapping it in a code fence. +_SOURCE_MARKDOWN_EXT = {"md", "mdx", "markdown", "qmd", "txt", ""} + +# Skip embedding a "source" larger than this (a mis-referenced binary/asset) +# so --with-sources never bloats the vault with megabytes of noise (#1968). +_MAX_SOURCE_BYTES = 2_000_000 + + +def _resolve_source_path(source_file: str, roots: list[Path]) -> "Path | None": + """Resolve a node's ``source_file`` to an existing readable file on disk. + + ``source_file`` is stored relative to the scan root (#1941), so we try each + candidate root in order; an absolute path is tried as-is first. Returns the + first existing regular file, or None when nothing resolves.""" + if not source_file: + return None + p = Path(source_file) + candidates: list[Path] = [] + if p.is_absolute(): + candidates.append(p) + else: + candidates.extend(r / source_file for r in roots) + for c in candidates: + try: + if c.is_file(): + return c + except OSError: + continue + return None + + +def _flatten_source_name(source_file: str) -> str: + """Flatten a (possibly nested, possibly Windows) source path into a single + collision-safe note stem prefixed ``_src_`` (e.g. ``en/best-practices.md`` + -> ``_src_en_best-practices``).""" + norm = source_file.replace("\\", "/") + parts = [seg for seg in norm.split("/") if seg not in ("", ".", "..")] + flat = "_".join(parts) if parts else "source" + # Drop a trailing markdown-ish extension so the note isn't "_src_x.md.md". + flat = re.sub(r"\.(md|mdx|qmd|markdown)$", "", flat, flags=re.IGNORECASE) + # Same unsafe-char strip the node filenames use, so wikilinks stay valid. + flat = re.sub(r'[\\/*?:"<>|#^[\]]', "", flat).strip() or "source" + return _cap_filename("_src_" + flat) + + +def _source_note_body(path: Path, source_file: str) -> "str | None": + """Build the markdown body for a copied source note, or None if the file is + unreadable or over the size cap. Markdown-like files are copied verbatim + (their own frontmatter/tags fall below our callout, so they no longer parse + as vault frontmatter); everything else is embedded in a code fence.""" + try: + if path.stat().st_size > _MAX_SOURCE_BYTES: + return None + except OSError: + return None + try: + text = path.read_text(encoding="utf-8", errors="replace") + except OSError: + return None + header = f"> [!note] Source file\n> `{_yaml_str(source_file).strip(chr(34))}`\n\n" + ext = path.suffix.lower().lstrip(".") + if ext in _SOURCE_MARKDOWN_EXT: + return header + text + lang = _SOURCE_FENCE_LANG.get(ext, ext) + return header + "```" + lang + "\n" + text.rstrip("\n") + "\n```\n" + + +def _write_source_notes(G: nx.Graph, roots: list[Path], owned_write) -> dict[str, str]: + """Copy every referenced, resolvable source file into ``sources/`` as one + note per distinct ``source_file`` and return a map ``source_file`` string -> + wikilink stem for the node notes to callout. Names are case-fold deduped so + two paths that flatten alike stay distinct on case-insensitive filesystems.""" + source_note_for: dict[str, str] = {} + used: set[str] = set() + for _node_id, data in G.nodes(data=True): + sf = data.get("source_file", "") + if not sf or sf in source_note_for: + continue + resolved = _resolve_source_path(sf, roots) + if resolved is None: + continue + body = _source_note_body(resolved, sf) + if body is None: + continue + base = _flatten_source_name(sf) + stem = base + n = 1 + while stem.lower() in used: + stem = f"{base}_{n}" + n += 1 + if owned_write(f"sources/{stem}.md", body): + used.add(stem.lower()) + source_note_for[sf] = stem + return source_note_for + + def to_obsidian( G: nx.Graph, communities: dict[int, list[str]], output_dir: str, community_labels: dict[int, str] | None = None, cohesion: dict[int, float] | None = None, + with_sources: bool = False, + source_root: "str | Path | list | tuple | None" = None, ) -> int: """Export graph as an Obsidian vault - one .md file per node with [[wikilinks]], plus one _COMMUNITY_name.md overview note per community (sorted to top by underscore prefix). @@ -464,6 +575,13 @@ def to_obsidian( Open the output directory as a vault in Obsidian to get an interactive graph view with community colors and full-text search over node metadata. + When ``with_sources`` is set (#1968), each node's ``source_file`` is copied + into ``sources/`` as a real note and every node note gets a callout linking + to it, so the vault is browsable standalone instead of every note reading as + a bare stub. ``source_file`` is stored relative to the scan root, so + ``source_root`` (default: the export's out dir and its parent) is where those + relative paths are resolved on disk. + Returns the number of node notes + community notes written. """ out = Path(output_dir) @@ -512,6 +630,21 @@ def safe_name(label: str) -> str: node_filename = _dedup_node_filenames(G, safe_name) + # --with-sources (#1968): copy each referenced source file into sources/ and + # remember which source_file strings got a note, so node notes can callout to + # them below. Roots to resolve the (scan-root-relative) source_file against: + # an explicit source_root, else the out dir and its parent (the default + # /graphify-out/ layout puts the scan root one level up). + source_note_for: dict[str, str] = {} + if with_sources: + if source_root is None: + roots = [out.parent, out] + elif isinstance(source_root, (list, tuple)): + roots = [Path(r) for r in source_root] + else: + roots = [Path(source_root)] + source_note_for = _write_source_notes(G, roots, _owned_write) + # Helper: compute dominant confidence for a node across all its edges def _dominant_confidence(node_id: str) -> str: confs = [] @@ -567,6 +700,19 @@ def _dominant_confidence(node_id: str) -> str: lines.append(f" - {tag}") lines += ["---", "", f"# {label}", ""] + # --with-sources: a callout linking this note to its copied source note, + # displayed with the original source_file path (#1968). + node_sf = data.get("source_file", "") + src_stem = source_note_for.get(node_sf) + if src_stem: + # A ']' or '|' in the display path would truncate the wikilink alias. + disp = node_sf.replace("|", "-").replace("[", "(").replace("]", ")") + lines += [ + "> [!info] Source", + f"> [[{src_stem}|{disp}]]", + "", + ] + # Outgoing edges as wikilinks neighbors = list(G.neighbors(node_id)) if neighbors: diff --git a/tests/test_obsidian_with_sources.py b/tests/test_obsidian_with_sources.py new file mode 100644 index 000000000..685fab60a --- /dev/null +++ b/tests/test_obsidian_with_sources.py @@ -0,0 +1,112 @@ +"""Tests for `--with-sources` on the Obsidian export (issue #1968). + +Without the flag, node notes are bare stubs (frontmatter + Connections). With +it, each node's `source_file` is copied into `sources/` as a real note and every +node note gets a callout linking to that source, so the vault is usable +standalone. +""" +import networkx as nx + +from graphify.export import to_obsidian + + +def _graph(): + G = nx.Graph() + # A markdown doc (copied verbatim) and a code file (fenced), plus a node + # whose source_file does not exist on disk (must get no callout). + G.add_node("n0", label="Best Practices", file_type="document", + source_file="en/best-practices.md", community=0) + G.add_node("n1", label="foo", file_type="code", source_file="util.py", community=0) + G.add_node("n2", label="Ghost", file_type="document", + source_file="missing/nope.md", community=0) + G.add_edge("n0", "n1", relation="mentions", confidence="EXTRACTED") + return G, {0: ["n0", "n1", "n2"]} + + +def _make_sources(root): + (root / "en").mkdir() + (root / "en" / "best-practices.md").write_text( + "---\ntitle: BP\n---\n# Best Practices\n\nUse graphify wisely.\n", encoding="utf-8") + (root / "util.py").write_text("def foo():\n return 42\n", encoding="utf-8") + + +def test_default_export_has_no_source_notes(tmp_path): + """Absent the flag, behaviour is unchanged: no sources/ dir, no callout.""" + _make_sources(tmp_path) + G, comms = _graph() + vault = tmp_path / "vault" + to_obsidian(G, comms, str(vault)) + assert not (vault / "sources").exists() + assert "> [!info] Source" not in (vault / "Best Practices.md").read_text(encoding="utf-8") + + +def test_with_sources_copies_and_wires_notes(tmp_path): + _make_sources(tmp_path) + G, comms = _graph() + vault = tmp_path / "vault" + to_obsidian(G, comms, str(vault), with_sources=True, source_root=tmp_path) + + # Source notes are copied under sources/ with collision-safe flattened names. + src_dir = vault / "sources" + assert (src_dir / "_src_en_best-practices.md").exists() + assert (src_dir / "_src_util.py.md").exists() + + # Markdown is copied verbatim; code is embedded in a fenced block. + md_src = (src_dir / "_src_en_best-practices.md").read_text(encoding="utf-8") + assert "Use graphify wisely." in md_src + py_src = (src_dir / "_src_util.py.md").read_text(encoding="utf-8") + assert "```python" in py_src and "return 42" in py_src + + # Each resolvable node note gets a callout linking to its source note, + # displayed with the original source_file path. + note = (vault / "Best Practices.md").read_text(encoding="utf-8") + assert "> [!info] Source" in note + assert "[[_src_en_best-practices|en/best-practices.md]]" in note + + +def test_unresolvable_source_gets_no_callout(tmp_path): + """A node whose source_file is missing on disk must not get a dangling + callout (and must not abort the export).""" + _make_sources(tmp_path) + G, comms = _graph() + vault = tmp_path / "vault" + to_obsidian(G, comms, str(vault), with_sources=True, source_root=tmp_path) + ghost = (vault / "Ghost.md").read_text(encoding="utf-8") + assert "> [!info] Source" not in ghost + assert not (vault / "sources" / "_src_missing_nope.md").exists() + + +def test_rerun_without_sources_prunes_source_notes(tmp_path): + """Source notes are graphify-owned, so dropping --with-sources on a re-export + prunes them and removes the callout — the vault returns to bare notes.""" + _make_sources(tmp_path) + G, comms = _graph() + vault = tmp_path / "vault" + to_obsidian(G, comms, str(vault), with_sources=True, source_root=tmp_path) + assert list((vault / "sources").glob("*.md")) + + to_obsidian(G, comms, str(vault), with_sources=False) + assert not list((vault / "sources").glob("*.md")) + assert "> [!info] Source" not in (vault / "foo.md").read_text(encoding="utf-8") + + +def test_source_root_list_and_fallback(tmp_path): + """source_root accepts a list of candidate roots (scan root, then out dir for + <=0.9.16 graphs); the first root that holds the file wins.""" + scan_root = tmp_path / "project" + out_dir = tmp_path / "project" / "graphify-out" + scan_root.mkdir() + out_dir.mkdir() + # source_file stored relative to the SCAN root (the #1941 layout). + (scan_root / "a.py").write_text("x = 1\n", encoding="utf-8") + G = nx.Graph() + G.add_node("n0", label="a", file_type="code", source_file="a.py", community=0) + G.add_node("n1", label="b", file_type="code", source_file="a.py", community=0) + G.add_edge("n0", "n1", relation="r", confidence="EXTRACTED") + vault = tmp_path / "vault" + to_obsidian(G, {0: ["n0", "n1"]}, str(vault), + with_sources=True, source_root=[scan_root, out_dir]) + # One shared source note for the shared source_file, both notes wired to it. + assert (vault / "sources" / "_src_a.py.md").exists() + assert "[[_src_a.py|a.py]]" in (vault / "a.md").read_text(encoding="utf-8") + assert "[[_src_a.py|a.py]]" in (vault / "b.md").read_text(encoding="utf-8")