diff --git a/.gitignore b/.gitignore index cdf5437..8eea27e 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ __pycache__/ .Python env/ venv/ +.venv/ ENV/ build/ develop-eggs/ @@ -37,3 +38,4 @@ wheels/ # Temporary files *.log *.tmp +maintainers-theme-evidence/ diff --git a/docs/about.md b/docs/about.md index c6d07d7..77112e7 100644 --- a/docs/about.md +++ b/docs/about.md @@ -1,25 +1,5 @@ # About -Personal site. ORNL is my employer; nothing here is an official ORNL product or statement. +I'm Joshua N. Grant, a geospatial systems architect at Oak Ridge National Laboratory, where I work on spatial data infrastructure, geospatial data warehouses, raster processing, and streaming systems. I came into software through plant-science research and data analysis, then worked in data engineering at ORNL and Bold Penguin before returning to ORNL in my current role. Outside work I build data tools, generative graphics, music systems, and small games — including PARQONAUT, NUMBRANE, and the other projects on [Projects](projects/index.md). This is my personal site and does not represent ORNL. -## Work - -Geospatial systems architect at **Oak Ridge National Laboratory** — spatial data warehouses, raster processing, streaming geospatial pipelines, and the Postgres/Kafka/object-storage stack around them. - -**Bold Penguin** (2022–2023): data engineering, Prefect 2 migration, CI/CD, services on Kubernetes. - -**Oak Ridge National Laboratory** (2019–2022): ETL on Kubernetes and Airflow, containerized services, COVID tracking data systems. - -## Outside work - -[Projects](projects/index.md): PARQONAUT, dotfiles, NUMBRANE, games, MIDI, and the rest. - -## Education - -MS and BS, plant sciences, University of Tennessee. Software came later, through data analysis and pipelines. - -## Contact - -[jngrant@live.com](mailto:jngrant@live.com) · [GitHub](https://github.com/sempervent) · [LinkedIn](https://linkedin.com/in/joshuanagrant) · [Blog](https://notjustadatum.blogspot.com) - -[Contact & Collaboration](getting-started.md) for phone and mailing address. +[Contact](getting-started.md) diff --git a/docs/adr/0001-site-architecture.md b/docs/adr/0001-site-architecture.md deleted file mode 100644 index 51a5bea..0000000 --- a/docs/adr/0001-site-architecture.md +++ /dev/null @@ -1,129 +0,0 @@ -# ADR-0001: Site Architecture — MkDocs + Material + Repo-as-Docs - -**Status**: Accepted - -**Date**: 2026-02-28 - -**Deciders**: Joshua N. Grant (site owner) - -**Technical Story**: Build a fast, searchable, maintainable documentation site that supports both professional best practices and playful experimentation — without a CMS, a backend, or a build team. - ---- - -## Context - -This site is a personal technical portfolio and documentation hub. It has two distinct audiences with different needs: - -1. **Engineers evaluating patterns** — want stable reference material, architectural rationale, and production-ready examples they can adapt. They browse by topic; they don't read linearly. -2. **Curious generalists** — stumble in via search or a link, want something interesting to read or build. They respond to personality and novelty. - -The site needed to satisfy both without becoming a bloated CMS or a hand-coded React app. Constraints: - -- Single author (no content team, no editorial workflow) -- Content lives in Git (version control, diff-ability, PR review for future collaborators) -- Must deploy to a free static host (GitHub Pages) -- Must be fast to build, cheap to maintain, and not require frontend engineering to update -- Must support full-text search without a search backend -- Must scale to hundreds of pages without navigation collapse - ---- - -## Decision - -**MkDocs** with the **Material for MkDocs** theme, deployed to **GitHub Pages** from this repository, with content organized under `docs/` as plain Markdown files. - -### Content domains - -Three top-level content domains with distinct contracts: - -| Domain | Purpose | Structure contract | -|---|---|---| -| **Best Practices** | Stable reference: patterns, governance, architecture | Conceptual; no assumed "do this right now" task | -| **Tutorials** | Task-oriented: step-by-step, copy-paste runnable | Prereqs → Steps → Verify → Troubleshoot | -| **Just for Fun** | Experimental: creative, playful, technically rigorous | No format constraint; must be reproducible | - -### Navigation philosophy - -- Tabs for top-level domains (`navigation.tabs`) -- Section grouping within tabs (`navigation.sections`) -- Breadcrumbs on every page (`navigation.path`) -- Long subsections (PostgreSQL, Docker) use nested subgroups — not flat 25-item lists -- Every major section has an `index.md` landing page - -### Discovery mechanisms - -- **Tags**: small, controlled vocabulary (≤ 15 tags total); rendered on a `/tags` index page -- **Search**: lunr-based full-text with symbol-aware separator (`[\s\-\_\.]+`); `search.suggest` and `search.share` enabled -- **What's New**: manually curated `whats-new.md`; updated when new content lands; no automation -- **See Also**: admonition blocks at the bottom of tutorial and best-practice pages; 2–3 links each; contextually adjacent only - -### Deployment - -GitHub Actions builds the site on push to `main` via `mkdocs gh-deploy`. No server. No CDN to configure. GitHub Pages serves static HTML. - ---- - -## Alternatives Considered - -### Hugo - -Fast at scale; single binary. Rejected: Go template syntax is a maintenance burden for a Markdown-first author. Material-equivalent theme quality requires significant front-end setup. No meaningful advantage at this site's scale. - -### Docusaurus (React) - -Strong MDX support; excellent for product docs with interactive components. Rejected: Node.js build pipeline; React authoring expected for advanced layouts; overkill for a site with no interactive components. Bundle size and build complexity add nothing here. - -### Sphinx (reStructuredText) - -Excellent for API reference documentation; first-class Python ecosystem. Rejected: reST markup is higher friction than Markdown for non-API content; theme ecosystem is weaker; search quality is inferior to Material's lunr integration. - -### GitBook / Notion / Confluence - -Managed SaaS tools with rich editors and collaboration features. Rejected: content is not Git-native; export lock-in; no control over URL structure; pricing; branding constraints. This site's content is code — it belongs in a code repository. - -### Single giant README or GitHub Wiki - -Minimal tooling; works for small projects. Rejected: no search, no nav structure, no code block copy buttons, no dark mode, no tagging, no per-section index pages. Collapses at scale. - ---- - -## Consequences - -### Positive - -- Zero server infrastructure; zero hosting cost -- Content is fully version-controlled; blame, diff, PR review all work -- MkDocs builds in < 15 seconds even at 300+ pages -- Material provides search, dark mode, code copy, admonitions, and tabs out of the box — no custom front-end work -- URL structure is stable (file path = URL); easy to cross-link and share -- `navigation.path` breadcrumbs solve "where am I?" without custom JS - -### Negative / Trade-offs - -- Nav hierarchy lives in `mkdocs.yml` YAML; verbose at scale and error-prone to maintain manually -- No dynamic content (no comments, no user accounts, no live search suggestions beyond lunr) -- Material plugin upgrades occasionally break config; `requirements.txt` must be pinned and maintained -- "What's New" is manual — if the author doesn't update it, it goes stale -- Tags are manual frontmatter — no auto-tagging - -### Follow-ups / Future Work - -- [ ] Automate `whats-new.md` generation from `git log` via a pre-commit hook or CI step -- [ ] Add `mkdocs-redirects` entries when pages are moved (plugin already installed) -- [ ] Evaluate `mkdocs-awesome-pages-plugin` to reduce YAML verbosity in nav -- [ ] Add a `projects/` section with structured project cards -- [ ] Revisit tag vocabulary as content grows; target ≤ 20 tags total - ---- - -## Operational Notes (guide future decisions) - -**URL preservation**: Never move a page without adding a redirect entry in `mkdocs.yml` via the `redirects` plugin. The plugin is already installed (`mkdocs-redirects`). A broken link from an external site is permanent damage. - -**Nav sanity**: Any subsection with more than 12 items should be split into named subgroups. The PostgreSQL section (26 items) is the canonical example — it was split into Core & Design, Performance & Operations, Advanced Features, and Integration & Deployment. - -**Cross-linking policy**: Every tutorial gets a `!!! tip "See also"` admonition with 2–3 links to contextually adjacent pages. Best-practice pages link to at least one tutorial that implements the pattern. Links must be relative and verified at build time. - -**Tag policy**: Tags are kept to a small, stable vocabulary. Before adding a new tag, check whether an existing tag covers it. Tags should reflect technology domains (`geospatial`, `postgresql`, `docker`), not content types or difficulty levels. - -**Content domain assignment**: When a page is ambiguous between Best Practices and Tutorials, assign it to the section that matches the primary reader intent. A page that readers open to *understand* goes in Best Practices. A page that readers open to *do something right now* goes in Tutorials. Cross-link between them. diff --git a/docs/adr/0002-why-mkdocs-material.md b/docs/adr/0002-why-mkdocs-material.md deleted file mode 100644 index e7917c5..0000000 --- a/docs/adr/0002-why-mkdocs-material.md +++ /dev/null @@ -1,83 +0,0 @@ -# ADR-0002: Use MkDocs + Material Theme for Documentation Site - -**Status**: Accepted - -**Date**: 2024-01-15 - -**Deciders**: Joshua N. Grant - -**Tags**: documentation, tooling, site - ---- - -## Context - -This site needed a documentation framework that could handle several hundred markdown files, support a rich nav hierarchy, provide fast full-text search, and deploy cleanly to GitHub Pages with zero server infrastructure. - -The author writes documentation in Markdown and wanted to keep authoring in Markdown — not learn a template DSL or manage a database. The site should build in CI, not require a local build step for every page edit, and look professional without heavy front-end engineering. - -## Decision - -Use **MkDocs** (the static site generator) with the **Material for MkDocs** theme. - -## Options Considered - -### Option 1: MkDocs + Material (chosen) - -**Pros:** -- Pure Markdown authoring; no shortcodes or template syntax needed -- Material theme provides tabs, search, code copy, dark mode, admonitions out of the box -- GitHub Pages deployment via `gh-pages` branch is first-class and well-documented -- `navigation.tabs`, `navigation.path`, `search.suggest`, `toc.integrate` cover all UX needs -- Plugin ecosystem: `git-revision-date-localized`, `tags`, `section-index`, `redirects` -- Active development; Material 9.x is stable and widely used - -**Cons:** -- Python dependency (minor; managed via `requirements.txt`) -- No JavaScript server-side rendering; purely static -- Large nav trees can produce long YAML in `mkdocs.yml` - -### Option 2: Hugo - -**Pros:** -- Faster builds at very large scale (thousands of pages) -- Single binary, no Python required - -**Cons:** -- Go template syntax is complex for non-Go authors -- Fewer out-of-the-box documentation UX patterns -- Material-equivalent theme quality requires more setup work - -### Option 3: Docusaurus (React) - -**Pros:** -- First-class MDX support (Markdown + JSX) -- Strong ecosystem for product documentation - -**Cons:** -- Node.js build pipeline; heavier dependency surface -- React component authoring expected for advanced layouts -- Overkill for a personal documentation site with no interactive components - -## Rationale - -MkDocs + Material is the most productive choice for a single-author, Markdown-first documentation site. Material provides the full UX feature set this site needs — tabs, search, dark mode, code copy, admonitions — with zero custom front-end work. Hugo is faster at extreme scale but adds template complexity that doesn't pay off here. Docusaurus is designed for product docs with interactive React components, which this site does not need. - -## Consequences - -### Positive -- Zero front-end engineering required to maintain the site -- Full-text search works out of the box with Material's lunr integration -- GitHub Actions CI builds and deploys in < 2 minutes - -### Negative -- Nav hierarchy is expressed in `mkdocs.yml` YAML, which becomes verbose at large scale -- Material requires specific plugin versions; upgrades occasionally break config - -### Neutral / Trade-offs -- Python version pinning matters for reproducible builds; `requirements.txt` must be maintained - -## Related Documents - -- [Creating MkDocs GitHub Site](../tutorials/quick-start/creating-mkdocs-github-site.md) -- [ADR and Technical Decision Governance](../best-practices/architecture-design/adr-decision-governance.md) diff --git a/docs/adr/0003-why-best-practices-vs-tutorials.md b/docs/adr/0003-why-best-practices-vs-tutorials.md deleted file mode 100644 index 62c0591..0000000 --- a/docs/adr/0003-why-best-practices-vs-tutorials.md +++ /dev/null @@ -1,87 +0,0 @@ -# ADR-0003: Separate "Best Practices" from "Tutorials" as Distinct Top-Level Sections - -**Status**: Accepted - -**Date**: 2024-02-01 - -**Deciders**: Joshua N. Grant - -**Tags**: information-architecture, documentation - ---- - -## Context - -As the site grew beyond 50 pages, two distinct types of content emerged with different authoring patterns and reader goals: - -1. **Conceptual/opinionated guides** — "How to think about X", pattern references, governance frameworks, architectural principles. Readers consult these when making decisions. They rarely run commands from these pages. - -2. **Step-by-step implementations** — "How to do X right now", with runnable commands, copy-paste configs, and clear success criteria. Readers follow these linearly to accomplish a specific task. - -Mixing both types under a single nav section forced readers to guess which kind of content they were getting before clicking. - -## Decision - -Maintain **two top-level sections**: - -- **Best Practices** — patterns, architectures, governance, opinionated guides. No assumed prerequisite of "do this exact thing right now." -- **Tutorials** — step-by-step, task-oriented, runnable. Has clear Prerequisites, Steps, Verify, and Troubleshoot structure. - -Cross-link between the two wherever a tutorial implements a best practice. - -## Options Considered - -### Option 1: Single "Documentation" section (flat) - -**Pros:** -- Simpler nav structure -- No category ambiguity - -**Cons:** -- Readers can't tell if a page is "read to understand" vs "follow to implement" -- Harder to maintain consistent page structure when types are mixed -- Harder to recommend "start here" paths for different reader goals - -### Option 2: Two sections: Best Practices + Tutorials (chosen) - -**Pros:** -- Mental model matches reader intent: "I want to understand" vs "I want to implement" -- Consistent page templates per section (best practice has Context/Pattern/Trade-offs; tutorial has Prereqs/Steps/Verify) -- Cross-links between sections surface the relationship between concept and implementation - -**Cons:** -- Some content genuinely fits both categories (e.g., a "best practices" page with runnable examples) -- Duplication risk if a best-practice and tutorial cover the same topic without linking to each other - -### Option 3: Tag-based (single section, discover by tag) - -**Pros:** -- No category assignment needed -- Tags can span multiple dimensions - -**Cons:** -- Requires robust tagging from day one (which doesn't exist at authoring time) -- Harder to build a coherent "recommended reading order" without section structure - -## Rationale - -The two-section model matches the dominant reader intents on this site. Engineers who know what they want to build open Tutorials. Engineers who are evaluating approaches open Best Practices. Keeping them separate preserves the ability to enforce different page templates per section and recommend distinct "Start Here" paths per audience. - -## Consequences - -### Positive -- Clear authoring expectations per section -- Separate "Start Here" recommendations per section (technical decision-makers vs implementors) -- Cross-links between sections create a richer content graph - -### Negative -- Some pages are genuinely ambiguous in category (handled by placing in the section that best matches the primary reader intent, then cross-linking) -- Nav depth increases because each section has its own subsections - -### Neutral / Trade-offs -- The "Just for Fun" section under Tutorials is an intentional exception: it's neither purely conceptual nor purely task-driven, but it belongs in Tutorials because readers follow it to build something - -## Related Documents - -- [ADR-0002: Why MkDocs + Material](0002-why-mkdocs-material.md) -- [Technical Documentation](../documentation.md) — site structure and navigation diff --git a/docs/adr/0004-why-just-for-fun-exists.md b/docs/adr/0004-why-just-for-fun-exists.md deleted file mode 100644 index 3c8cd0f..0000000 --- a/docs/adr/0004-why-just-for-fun-exists.md +++ /dev/null @@ -1,94 +0,0 @@ -# ADR-0004: Include a "Just for Fun" Section in a Technical Documentation Site - -**Status**: Accepted - -**Date**: 2024-03-10 - -**Deciders**: Joshua N. Grant - -**Tags**: documentation, information-architecture - ---- - -## Context - -A personal technical documentation site that presents only enterprise patterns and production best practices risks becoming indistinguishable from thousands of other engineering blogs. It signals "I can execute corporate patterns" but not "I explore ideas at the edges of what's technically possible." - -The site author builds projects that don't fit neatly into "Best Practices" or "Tutorials" — things like: - -- A Raspberry Pi sample library server with USB MIDI live audition -- Recursive cathedral generators from L-system grammars -- Metrics sonification via SuperCollider + OSC -- WebGL generative art from PostGIS raster data - -These projects are technically rigorous, reproducible, and often more instructive about system design than a standard tutorial — but their primary motivation is curiosity, not production deployment. - -## Decision - -Create a **"Just for Fun" subsection** under Tutorials. Include creative, experimental, and exploratory projects that are: - -- Technically reproducible (not just concept posts) -- Non-trivial in implementation (demonstrate real engineering) -- Built for intrinsic interest, not business requirements - -Do not artificially restrict the section to specific technology areas. Let it grow organically. - -## Options Considered - -### Option 1: Exclude creative projects entirely - -**Pros:** -- Site maintains a purely professional tone -- No reader confusion about what the site is "for" - -**Cons:** -- Loses the most distinctive content on the site -- The best projects — the ones that demonstrate creative problem-solving — get no home -- Portfolio signals only "follows conventional patterns," not "thinks originally" - -### Option 2: Separate blog/personal site for fun projects - -**Pros:** -- Clean separation of concerns -- Professional docs site stays focused - -**Cons:** -- Two sites to maintain; cross-traffic is lost -- The technical depth of these projects is documentation-worthy, not blog-post-worthy -- Discoverability drops: readers who arrive for best practices never discover the creative work - -### Option 3: "Just for Fun" section within the documentation site (chosen) - -**Pros:** -- Readers who arrive for serious content discover unexpected creative depth -- The same documentation framework serves both content types -- Cross-links between fun projects and related best practices reinforce conceptual connections (e.g., Pi sample server → Docker best practices) -- Signals that the author is a whole engineer, not just a pattern-follower - -**Cons:** -- Some readers may find the tone inconsistent with the rest of the site -- Section needs its own voice: playful but technically precise, not corporate - -## Rationale - -The "Just for Fun" section is a differentiator, not a distraction. Every page in the section demonstrates real engineering — recursive L-systems, WebAudio API internals, MIDI protocol handling, SQLite query optimization. The playful framing makes these approachable; the technical depth makes them credible. Excluding this content would make the site less interesting and less representative of how the author actually works. - -## Consequences - -### Positive -- Site has a unique character that distinguishes it from generic engineering blogs -- Creative projects get the same documentation quality as production content -- Cross-links from fun projects into best practices and tutorials create unexpected discovery paths - -### Negative -- Section requires active curation to maintain quality bar (reproducible, non-trivial) -- Voice calibration is harder: playful but not cringe; experimental but not hand-wavy - -### Neutral / Trade-offs -- "Managing People in Software Development" is in this section despite being non-technical — it fits the spirit of "unexpected but useful" content that defines the section - -## Related Documents - -- [Just for Fun Overview](../tutorials/just-for-fun/index.md) -- [ADR-0003: Best Practices vs Tutorials](0003-why-best-practices-vs-tutorials.md) -- [Technical Documentation](../documentation.md) — site structure and navigation diff --git a/docs/adr/0005-esp32-section-architecture.md b/docs/adr/0005-esp32-section-architecture.md deleted file mode 100644 index 73429f3..0000000 --- a/docs/adr/0005-esp32-section-architecture.md +++ /dev/null @@ -1,96 +0,0 @@ -# ADR 0005: Establish ESP32 & Embedded Systems as a First-Class Documentation Section - -- **Status**: Accepted -- **Date**: 2026-02-26 -- **Deciders**: mate (owner) -- **Technical Story**: ESP32-based projects have accumulated across the site without a coherent home. Embedded systems combine hardware, firmware, electrical safety, and security in ways that require a dedicated, structured documentation pillar — not a footnote under "just for fun." - ---- - -## Context - -The site already has stable pillars for server-side concerns (PostgreSQL, Docker, Python, Rust, Go) and creative projects (Just for Fun). But embedded systems occupy a gap: - -- Hardware and software decisions are deeply coupled — you cannot explain firmware without explaining the circuit. -- Electrical safety is non-negotiable and must be part of the documentation, not an afterthought. -- ESP32 projects span best practices (architecture, power, security) *and* tutorials (capstone builds), requiring both documentation modes. -- Without a dedicated section, embedded content risks scattering across "Just for Fun" (too casual), "Docker" (wrong stack), or nowhere at all. - -Specific drivers: -1. Growing number of ESP32 personal projects (sensor nodes, room controllers, art frames). -2. Need to document safety-first patterns that apply across all embedded projects, not just one tutorial. -3. Desire to distinguish *durable practices* (deep sleep design, ISR safety, power budgets) from *playful experiments* (e-ink art frames). -4. Embedded docs must cross-link hardware ↔ firmware ↔ security — a flat structure cannot represent these dependencies well. - ---- - -## Decision - -**Create `Best Practices → 🔌 Embedded Systems & ESP32`** as a first-class section at the same level as Python, Docker, and PostgreSQL, with the following structure: - -| Content type | Location | -|---|---| -| Durable patterns (architecture, power, safety, security, sensors, display) | `docs/best-practices/esp32/` | -| Applied project tutorials | `docs/tutorials/embedded/` | -| Architectural rationale (this ADR) | `docs/adr/` | - -**Safety as mandatory content**: every best practice document in this section must address safety implications — electrical, power, or security — as a primary concern, not a sidebar. - -**Cross-linking policy**: every tutorial references the relevant best practice pages via `!!! tip "See also"` admonitions. Every best practice page links back to at least one applied tutorial. - -**Naming convention**: `kebab-case.md`, `esp32-` prefix for ESP32-specific docs, `embedded/` for the tutorial subdirectory. - ---- - -## Alternatives Considered - -### Keep ESP32 under Just for Fun only - -Rejected. "Just for Fun" signals low stakes and experimentation. Electrical safety documentation, power budget analysis, and OTA security do not belong under that framing. A reader looking for grounded hardware guidance would not find it. - -### Generic "Hardware" catch-all section - -Rejected. "Hardware" is too broad. The site has no Raspberry Pi HAT tutorials, no PCB design content, no STM32 content — a generic section would be an empty promise. ESP32 is the concrete, bounded scope that exists today. - -### One giant "ESP32 reference" page - -Rejected. A monolithic page cannot represent the relationship between topics (e.g., "power gating depends on both the circuit design page and the firmware architecture page"). Separate, cross-linked pages enable navigation by reader intent. - -### Separate repo - -Rejected. The value of co-locating embedded content with Docker, Go, and security content is cross-pollination: a reader learning about OTA updates should easily reach the general secrets management guide. Splitting the repo breaks that graph. - ---- - -## Consequences - -### Positive - -- Clear separation between durable patterns (best practices) and applied projects (tutorials) — consistent with [ADR-0003](0003-why-best-practices-vs-tutorials.md) -- Safety documentation is guaranteed visible and linked from every project page -- Embedded becomes a stable, growing pillar alongside server-side content -- Cross-links between embedded tutorials and Docker/Security/Kubernetes content are natural and short -- New contributors have a clear template to follow for future embedded topics - -### Negative / Trade-offs - -- Increases nav surface area by ~10 entries; sidebar grows slightly longer -- Structural complexity: readers must understand that "programming architecture" is in best-practices, while "RF controller" is in tutorials -- Requires maintaining cross-links as new content is added (manageable with the See Also convention) - -### Follow-ups - -- [ ] Add LoRa best practices page (SX1276/SX1278, spreading factor, duty cycle limits) -- [ ] Add power electronics section (MOSFET switches, LiPo charging circuits, solar input) -- [ ] Add printable ESP32 safety checklist (HTML/PDF export from Markdown) -- [ ] Add ESP32-S3 and ESP32-C3 notes (USB native, RISC-V, BLE 5.0 differences) -- [ ] Consider an `embedded/` best-practices subdirectory if non-ESP32 MCUs are added (STM32, RP2040) - ---- - -## Notes - -- Follow the [URL preservation policy from ADR-0001](0001-site-architecture.md): if any embedded page is ever moved, add a redirect. -- Nav rule: if the embedded section exceeds 12 items, group into sub-sections (e.g., "Core", "Communication", "Power"). -- Tag policy: use `embedded` and `esp32` tags consistently; add `security` or `performance` where appropriate. -- This ADR does not cover non-ESP32 microcontrollers. A future ADR should decide whether to expand the section or create a sibling section. diff --git a/docs/adr/0006-embedded-expansion-lora-power.md b/docs/adr/0006-embedded-expansion-lora-power.md deleted file mode 100644 index 86f5784..0000000 --- a/docs/adr/0006-embedded-expansion-lora-power.md +++ /dev/null @@ -1,79 +0,0 @@ -# ADR 0006: Expand Embedded Section — LoRa, Power Electronics, and Variant Notes - -- **Status**: Accepted -- **Date**: 2026-02-26 -- **Deciders**: mate (owner) -- **Supersedes**: N/A — extends [ADR-0005](0005-esp32-section-architecture.md) - ---- - -## Context - -[ADR-0005](0005-esp32-section-architecture.md) established the `best-practices/esp32/` section and committed to follow-ups: LoRa best practices, power electronics, a printable safety checklist, and ESP32-S3/C3 notes. - -Implementation revealed a structural decision: **Power Electronics** does not fit cleanly under `esp32/`. The content (MOSFET switching, LiPo charging circuits, solar, buck vs LDO) applies to any embedded microcontroller — not only ESP32. It belongs at a higher level of abstraction. - -This ADR formalizes: -1. The creation of `best-practices/embedded/` as a cross-cutting hardware folder -2. The placement of LoRa, printable checklist, and S3/C3 notes under `esp32/` -3. The nav grouping for `🔋 Power Electronics & Embedded Hardware` - ---- - -## Decision - -**Create `docs/best-practices/embedded/`** as a sibling to `best-practices/esp32/`, `best-practices/docker-infrastructure/`, etc. - -| Content | Location | Rationale | -|---|---|---| -| Power Electronics (MOSFETs, LiPo, solar, buck/LDO) | `best-practices/embedded/` | Platform-agnostic hardware concerns | -| LoRa best practices (SX127x) | `best-practices/esp32/` | ESP32-centric driver and firmware guidance | -| Printable safety checklist | `best-practices/esp32/` | ESP32-specific checklist; referenceable from hardware doc | -| ESP32-S3/C3 notes | `best-practices/esp32/` | Variant comparison for ESP32 ecosystem | - -**Nav grouping**: `🔋 Power Electronics & Embedded Hardware` appears in mkdocs.yml alongside `🔌 Embedded Systems & ESP32` under Best Practices. - -**Cross-linking policy**: the `embedded/` folder links back to `esp32/` for firmware context; `esp32/` pages link to `embedded/` for hardware context. - ---- - -## Alternatives Considered - -### Put power electronics under `esp32/` - -Rejected. A page titled "Power Electronics for ESP32" is still useful to Raspberry Pi users, STM32 users, and Arduino users. Placing it under `esp32/` would mislead the IA and make it harder to find for non-ESP32 readers. The content's value is platform-agnostic. - -### Create a `hardware/` folder instead of `embedded/` - -Rejected. "Hardware" is a broader signal than intended. "Embedded" is more accurate — it connotes electronics that host a microcontroller, which is precisely the scope. - -### Single `embedded/` folder replacing `esp32/` - -Rejected. The `esp32/` folder contains firmware-level guidance (FreeRTOS, ISRs, deep sleep API) that is ESP32-specific. Merging would dilute the precision of that content. - -### No new folder — just add power electronics to `esp32/` - -Rejected — see first alternative. - ---- - -## Consequences - -### Positive - -- Power electronics guidance is reachable by readers working on any embedded platform -- Clear separation: `esp32/` = ESP32 firmware + peripherals; `embedded/` = universal hardware concerns -- Sets a precedent for future MCU-agnostic content (RP2040, STM32, AVR) to have a home without needing to dilute the ESP32 section -- Printable safety checklist is directly reachable from the nav and from hardware build references - -### Negative / Trade-offs - -- Nav surface area increases by one section and two entries -- A reader unfamiliar with the split may look in the wrong section first — mitigated by cross-links - -### Follow-ups - -- [ ] Add RP2040 / Raspberry Pi Pico notes to `embedded/` if Pico projects are documented -- [ ] Add STM32 power notes if STM32 tutorials are added -- [ ] Consider `embedded/antenna-and-rf-layout.md` for shared RF layout guidance across LoRa, WiFi, BLE -- [ ] Evaluate whether the printable checklist would benefit from MkDocs print CSS customization diff --git a/docs/adr/0007-deep-dives-section.md b/docs/adr/0007-deep-dives-section.md deleted file mode 100644 index c326512..0000000 --- a/docs/adr/0007-deep-dives-section.md +++ /dev/null @@ -1,95 +0,0 @@ -# ADR 0007: Introduce "Deep Dives" Section for Analytical Long-Form Content - -- **Status**: Accepted -- **Date**: 2026-02-26 -- **Deciders**: mate (owner) -- **Related**: [ADR-0003](0003-why-best-practices-vs-tutorials.md) (Best Practices vs Tutorials distinction) - ---- - -## Context - -The site has two content modes: prescriptive (Best Practices) and procedural (Tutorials). Both are optimized for practical output — a reader leaves with a configuration, a working build, or a mental model of "what to do." - -Some subjects resist this framing. They are not best solved by prescription or procedure. They require comparison, historical context, regulatory analysis, threat modeling, and architectural argumentation. Examples already emerging in the content: - -- LoRaWAN vs raw LoRa: a decision shaped by spectrum law, security assumptions, infrastructure cost, and fleet size -- MQTT vs CoAP vs HTTP for IoT: protocol choice determined by transport semantics, not code examples -- Scratch vs distroless: not a tutorial topic; a philosophical and operational stance - -Attempting to contain these analyses in a Best Practice page produces either a superficial treatment or an overwhelming document that violates the section's prescriptive purpose. Containing them in a tutorial adds irrelevant instructional scaffolding to what is fundamentally an argument. - -The existing section structure does not provide a natural home for analytical, argument-driven content. - ---- - -## Decision - -Create `docs/deep-dives/` as a top-level content section with its own nav tab: **Deep Dives**. - -Content placed here must satisfy all of the following: - -- **Comparative**: examines two or more approaches, protocols, architectures, or tools in genuine tension -- **Analytical**: presents reasoning and evidence, not instructions -- **Argument-driven**: arrives at a defensible position or decision framework -- **Less code-heavy**: code snippets are illustrative, never the primary content -- **Intentional**: a deep dive is written because the topic *demands* depth — not as a way to expand the site - -Content that is prescriptive belongs in Best Practices. Content that is procedural belongs in Tutorials. Content that is analytical and comparative belongs in Deep Dives. - -**Placement**: top-level nav between Best Practices and Tutorials. Deep Dives is neither a reference nor a guide — it is a section a reader visits to think, not to do. - ---- - -## Alternatives Considered - -### Overload Best Practices with long-form comparisons - -Rejected. Best Practices are prescriptive by contract (see ADR-0003). A 4000-word analysis of LoRaWAN vs raw LoRa is not a best practice — it is a decision support document. Mixing the two dilutes the purpose of the Best Practices section and makes it harder to navigate. - -### Write excessively long tutorials - -Rejected. Tutorials are procedural. Adding comparative analysis to a tutorial inflates it and frustrates readers who came to build something. A reader who wants to understand the LoRaWAN security model is not the same reader who wants firmware wiring instructions. - -### External blog (separate from the site) - -Rejected. A separate blog creates friction: different URL, different deployment, different search index, different navigation context. The goal is to keep high-signal content co-located with the technical reference material it references, enabling cross-linking and unified search. - -### One overloaded "analysis" page per topic area - -Rejected. A single "Embedded Systems Analysis" page containing every comparative discussion would be incoherent and unsearchable. Individual, focused deep dives are more discoverable and maintainable. - ---- - -## Consequences - -### Positive - -- Provides a legitimate outlet for analytical depth without corrupting the purpose of existing sections -- Raises the intellectual surface area of the site — a reader can move from "how to configure a LoRa transmitter" (Best Practices) to "why LoRaWAN might be wrong for your deployment" (Deep Dives) within the same site -- Encourages higher-quality writing: the analytical mode demands more careful argumentation than a checklist -- Cross-linking between Deep Dives and Best Practices / Tutorials creates a richer content graph - -### Negative / Trade-offs - -- Increases nav surface area by one top-level tab -- Requires editorial discipline: Deep Dives must remain intentional and rare — if every topic gets a deep dive, the section loses its signal -- Analytical writing takes longer to produce and review than prescriptive writing - ---- - -## Operational Rules - -1. A Deep Dive is created only when the topic genuinely resists prescription or procedure -2. Each Deep Dive must cross-link to at least one Best Practice or Tutorial page -3. Deep Dives do not include step-by-step wiring instructions, setup commands, or "how to install X" -4. Style: analytical, comparative, journalistic — not tutorial, not reference -5. If a Deep Dive eventually produces a clear prescription, extract that prescription into a Best Practice page and link back - ---- - -## Follow-ups - -- [ ] Add a style guide for Deep Dives (tone, structure, code usage policy) as a meta-page in the section -- [ ] Candidate topics: MQTT vs CoAP vs HTTP for IoT, Scratch vs distroless container philosophy, ADR-driven architecture decision culture, LoRaWAN vs Zigbee vs Thread for smart home -- [ ] Evaluate whether Deep Dives warrant their own RSS feed or tag filtering diff --git a/docs/adr/0008-expand-deep-dives-section.md b/docs/adr/0008-expand-deep-dives-section.md deleted file mode 100644 index 2d11f29..0000000 --- a/docs/adr/0008-expand-deep-dives-section.md +++ /dev/null @@ -1,100 +0,0 @@ ---- -tags: - - adr - - documentation ---- - -# ADR 0008: Expand Deep Dives as a First-Class Analytical Section - -**Status**: Accepted -**Date**: 2026-02-26 -**Author**: Joshua N. Grant - ---- - -## Context - -The site's Deep Dives section was established in ADR 0007 with a single entry (LoRaWAN vs Raw LoRa) as a proof of concept for long-form analytical content. That entry demonstrated the format's value: structured comparative analysis, trade-off matrices, historical context, and decision frameworks — distinct from both tutorials and best-practice checklists. - -Several recurring content needs within this site's scope are poorly served by existing content types: - -- **Format and storage decisions** involve trade-offs that span years of ecosystem evolution and cannot be reduced to a checklist. -- **System selection decisions** (databases, engines, protocols) require comparative analysis across dimensions that change with organizational scale. -- **Infrastructure philosophy** (container base images, infrastructure automation) involves trade-offs between correctness, operational complexity, and security that are contextual rather than prescriptive. - -These topics warrant a separate content type with its own structural guardrails. - ---- - -## Decision - -Expand the Deep Dives section from one entry to eight, adding: - -1. **Parquet vs CSV vs ORC vs Avro** — columnar storage theory, compression, predicate pushdown, ecosystem analysis -2. **Lakehouse vs Warehouse vs Database** — historical arc, table format emergence, governance complexity, cost structures -3. **DuckDB vs PostgreSQL vs Spark** — execution models, data gravity, hybrid architecture -4. **Geospatial File Format Choices** — raster vs vector, Cloud-Optimized GeoTIFF, GeoParquet emergence -5. **Container Base Image Philosophy** — glibc vs musl, attack surface analysis, operational debugging cost -6. **Infrastructure as Code vs GitOps** — control plane models, drift detection, hybrid governance -7. **MQTT vs HTTP in IoT Systems** — connection models, power consumption, broker centralization, security - -These entries are organized under four sub-sections in the navigation: - -- Data Formats & Storage -- Data Systems & Architecture -- Infrastructure & Automation -- Embedded & Radio - ---- - -## Why Analytical Essays Deserve Structural Separation - -Best practices pages answer the question: "given that I am doing X, how should I do it well?" Tutorials answer: "how do I accomplish X step by step?" - -Deep dives answer a different question: "should I be doing X at all, and under what conditions does X beat Y?" This requires a different structure — comparative analysis, historical context, trade-off matrices — and a different tone: analytical and opinionated rather than prescriptive and procedural. - -Mixing these content types in the same section would dilute both. Best practices readers want concrete guidance; deep dive readers want structured argument. Structural separation preserves the value of each. - ---- - -## Guardrails Against Content Sprawl - -Deep dives carry an expansive mandate ("analytical exploration") that can attract poorly scoped content. The following criteria apply to all deep dive candidates: - -**A deep dive is appropriate when**: -- The topic involves a genuine architectural or technology selection decision with non-obvious trade-offs -- The decision consequences extend across months or years of system operation -- Comparative analysis across at least two options is possible and meaningful -- The topic cannot be adequately addressed in a best-practice page without becoming a comparative essay - -**A deep dive is not appropriate when**: -- The correct answer is well-established and non-controversial -- The topic is better served by a tutorial (step-by-step) or best-practice (how-to-well) -- The topic is too narrow to warrant the deep dive structure (decision framework, trade-off matrix, historical context) -- The topic duplicates an existing deep dive without adding a meaningfully different perspective - ---- - -## Criteria for New Deep Dives - -New deep dives should meet all of the following before being added: - -1. **Genuine tension**: two or more approaches with legitimate use cases — not a comparison where one option is obviously correct -2. **Consequential decision**: a choice that affects system architecture, data portability, operational complexity, or long-term maintainability -3. **Analytical completeness**: the entry must include a historical or contextual arc, at least one comparative matrix, ASCII diagram(s) where clarifying, and a decision framework section -4. **Appropriate length**: 1,000–3,000 words; shorter entries should be best practices; longer entries should be split into multiple deep dives - ---- - -## Consequences - -The Deep Dives section becomes a stable, growing analytical corpus alongside tutorials and best practices. Navigation is organized by domain (formats, systems, infrastructure, embedded) rather than by chronological addition. Cross-links between related deep dives and related tutorials/best practices are maintained as part of the authoring checklist. - -The risk of content sprawl is managed by the criteria above. The ADR process itself (this document) serves as the governance mechanism for section-level structural changes. - ---- - -## Related Documents - -- [ADR 0007: Deep Dives Section](0007-deep-dives-section.md) — original decision to create the section -- [ADR 0003: Best Practices vs Tutorials](0003-why-best-practices-vs-tutorials.md) — the content type distinction that deep dives extend diff --git a/docs/adr/0009-deep-dives-expansion.md b/docs/adr/0009-deep-dives-expansion.md deleted file mode 100644 index ee7e99e..0000000 --- a/docs/adr/0009-deep-dives-expansion.md +++ /dev/null @@ -1,91 +0,0 @@ ---- -tags: - - adr - - documentation ---- - -# ADR 0009: Expand Deep Dives into Core Analytical Layer - -**Status**: Accepted -**Date**: 2026-02-26 -**Author**: Joshua N. Grant - ---- - -## Context - -ADR 0008 formalized the expansion of Deep Dives from one entry (LoRaWAN vs Raw LoRa) to eight entries, establishing structural sub-sections and criteria for new entries. This document addresses a further expansion: four additional deep dives covering data pipeline reliability, observability architecture, embedded platform selection, and workflow orchestration philosophy. - -The expansion brings the Deep Dives section to twelve entries across five sub-sections (Data Formats & Storage, Data Systems & Architecture, Operations & Reliability, Infrastructure & Automation, Embedded & Radio). This constitutes a material increase in scope — the section is no longer a collection of isolated analytical essays but an interconnected analytical corpus with deliberate cross-linking and a consistent epistemic standard. - ---- - -## Decision - -Accept the following four deep dives into the corpus: - -1. **Why Most Data Pipelines Fail** — organizational and architectural failure modes, schema drift, ownership ambiguity, the CI/CD gap in data systems, and survivable architectural patterns -2. **Observability vs Monitoring** — monitoring vs observability semantics, three pillars analysis, cardinality economics, data pipeline observability, cultural factors, tooling landscape -3. **ESP32 vs Raspberry Pi** — microcontroller determinism vs Linux convenience, power profiles, security surface, ecosystem maturity, and deployment decision matrix -4. **Prefect vs Airflow** — DAG-first vs Python-native orchestration philosophy, execution model architecture, failure semantics, scaling models, and operational burden - -Add a new sub-section to the nav: **Operations & Reliability** (currently containing the Observability vs Monitoring deep dive, with room for additional entries in the reliability and incident management domain). - ---- - -## Why These Are Not Blog Posts - -Blog posts are timely, personal, and conversational. They may be opinionated without systematic justification, and they do not typically include formal decision frameworks or comparative analyses. - -Deep Dives in this corpus are: - -- **Structured**: every entry follows a consistent schema (thesis, context, analysis, comparison, decision framework) that enables comparative reading across entries -- **Durable**: the analysis is written to remain relevant across years, not weeks; where technology changes rapidly, the structural principles rather than specific tools are the primary focus -- **Referenced**: cross-links to related deep dives, best practices, and tutorials make each entry part of a navigable analytical corpus -- **Analytical**: positions are taken and justified with evidence and logical argument, not asserted as opinions -- **Non-promotional**: vendor mentions are comparative and contextual, not endorsements - -The test question: would this content belong in an engineering organization's internal knowledge base, cited in architecture review documents? If yes, it is a deep dive. If it reads like a personal take on a recent controversy, it is a blog post. - ---- - -## Criteria for Inclusion - -A proposed deep dive must satisfy all of the following: - -**1. Genuine and consequential tension**: the topic must involve a real architectural decision with non-obvious trade-offs and meaningful consequences for system longevity, team capability, or operational cost. "Which library should I use for X?" is not a deep dive topic. "What does the choice of orchestration model imply for organizational ownership of data pipelines?" is. - -**2. Analytical completeness**: the entry must include at minimum: a historical or contextual arc, a comparative analysis of two or more approaches, at least one structured comparison (table, matrix, or diagram), and a decision framework that provides actionable guidance. - -**3. Appropriate scope**: a deep dive should address a single coherent question or tension. If the scope requires more than 3,000 words to be analytically complete, it is likely too broad and should be split. If it can be addressed in fewer than 800 words, it is likely a best practice annotation, not a deep dive. - -**4. Durability**: the analysis should remain relevant for at least two to three years without significant revision. Technology-specific entries (e.g., "Airflow 2.6 vs 2.7") fail this criterion. Platform-philosophy entries (e.g., "DAG-first vs code-first orchestration models") satisfy it. - -**5. Non-overlap with existing content**: a proposed deep dive must not substantially duplicate an existing deep dive. It may reference, contrast, or extend existing deep dives, but the primary analytical contribution must be distinct. - ---- - -## Guardrails Against Dilution - -The Deep Dives section degrades in analytical value if it accumulates: - -- Entries that are tutorials in analytical clothing (detailed implementation instructions with light framing) -- Entries that are opinion pieces without analytical structure -- Entries on topics so narrow that they would better serve as best practice annotations -- Entries on topics so broad that they cannot be addressed rigorously in a single document -- Entries duplicating content from existing deep dives without a meaningfully distinct analytical contribution - -The ADR process serves as the formal check on section-level structural changes (adding new sub-sections, reorganizing the corpus). Individual deep dive entries do not require an ADR if they meet the criteria above. New sub-sections or reorganizations require an ADR update. - ---- - -## Consequences - -The Deep Dives section becomes a twelve-entry analytical corpus with five sub-sections. Cross-links between related entries are maintained as part of authoring discipline. The "Operations & Reliability" sub-section is established as a home for future entries addressing failure analysis, incident management, and reliability engineering at the conceptual level. - ---- - -## Related Documents - -- [ADR 0008: Expand Deep Dives as a First-Class Analytical Section](0008-expand-deep-dives-section.md) — previous expansion and criteria formalization -- [ADR 0007: Deep Dives Section](0007-deep-dives-section.md) — original section creation diff --git a/docs/adr/0010-deep-dives-governance.md b/docs/adr/0010-deep-dives-governance.md deleted file mode 100644 index f9ea2e0..0000000 --- a/docs/adr/0010-deep-dives-governance.md +++ /dev/null @@ -1,126 +0,0 @@ ---- -tags: - - adr - - documentation ---- - -# ADR 0010: Governance Model for Deep Dives - -**Status**: Accepted -**Date**: 2026-02-26 -**Author**: Joshua N. Grant - ---- - -## Context - -The Deep Dives section has grown to seventeen entries across six sub-sections through ADRs 0007–0009. Three successive expansion ADRs have each restated criteria for inclusion with minor variations. This ADR consolidates the governance model into a single authoritative reference, superseding the inclusion criteria statements in ADRs 0008 and 0009. - -The growth of the section raises a specific risk: as the entry count increases, the average quality and analytical rigor of entries tends to decline. The first entries in any analytical corpus establish a standard that becomes harder to maintain as the pressure to fill in topics increases. Explicit governance makes the standard visible and enforceable. - ---- - -## Why Long-Form Analytical Content Requires Curation - -Long-form analytical content is not self-governing. The absence of curation produces a set of tendencies that degrade corpus quality over time: - -**Scope inflation**: without defined scope, entries expand to cover everything tangentially related to the topic. An entry that begins as an analysis of distributed tracing trade-offs expands to include step-by-step setup instructions, vendor comparison tables, and configuration recommendations. The analytical character is diluted by operational detail that belongs in best practices or tutorials. - -**Tone drift**: without a defined tone standard, analytical entries drift toward tutorial voice (instructional), blog voice (conversational and opinionated without systematic justification), or vendor-advocacy voice (promotional). Each is appropriate in a different content type; none is appropriate for this section. - -**Redundancy**: without cross-reference awareness, entries duplicate analysis that already exists in the corpus. The economics of observability overlap with the observability vs monitoring entry; the appropriate use of microservices overlaps with the monolith entry. Redundancy is acceptable if the analytical angle is genuinely distinct; it is wasteful if it is merely repetition. - -**Obsolescence**: analytical content on rapidly changing technologies can become misleading without update. An entry on a specific tool's architecture that was accurate in 2024 may be incorrect by 2026. The governance model must include a policy for handling obsolescence. - ---- - -## Criteria for Inclusion - -A proposed deep dive must satisfy all criteria in each category. - -### Topic Criteria - -1. **Genuine architectural tension**: the topic involves a decision with consequential trade-offs that cannot be resolved by a simple rule. Decisions where one option is obviously correct for all contexts do not merit deep dive treatment. - -2. **Consequential scope**: the decision affects system longevity, team capability, operational cost, or organizational structure across a time horizon of at least one to two years. Short-term implementation decisions belong in tutorials or best practices. - -3. **Non-duplication**: the proposed entry must not substantially duplicate an existing entry. A different analytical angle on the same topic is acceptable if the angle is explicitly identified and the new entry cross-references the existing one. Mere restatement is not acceptable. - -4. **Durability**: the core analysis must remain relevant for two to three years without requiring substantive revision. Tool-specific version comparisons and feature checklists fail this criterion. Philosophy-level comparisons of execution models, governance approaches, or architectural trade-offs satisfy it. - -### Structural Criteria - -Every deep dive must include, in order: - -1. **Opening Thesis** — a clear statement of the analytical claim the document supports. Not "this document discusses X"; a substantive claim about X that the document then argues for. - -2. **Historical Context** — the conditions that produced the current situation, the predecessor approaches, and why the current alternatives exist. - -3. **Analytical Core** — the comparison, trade-off analysis, or architectural examination. This section should include at least one structured comparison (table, matrix, or ASCII diagram) and at least two to three paragraphs of dense analytical prose per major claim. - -4. **Decision Framework** — actionable guidance organized by stakeholder context (team size, regulatory environment, operational maturity, workload characteristics). Not "it depends" without criteria; specific criteria for each branch of the decision. - -5. **Cross-links** — at minimum two links to related content in the site (deep dives, best practices, tutorials, ADRs). Cross-links are not decorative; they should reflect genuine analytical relationships. - -### Tone Criteria - -All deep dives must: - -- Avoid tutorial voice: no step-by-step instructions, no "to do X, first do Y" sequences -- Avoid marketing voice: no vendor endorsements, no superlative claims without evidence, no promotional language for any tool or platform -- Avoid blog voice: positions must be argued with evidence and logical structure, not asserted as personal opinions -- Avoid hedging to the point of uselessness: the Decision Framework must make actual recommendations, not merely list considerations - ---- - -## Sub-Section Governance - -New sub-sections may be created when at least two deep dives share a coherent analytical domain that is not adequately captured by existing sub-sections. A single entry does not justify a new sub-section. Sub-section creation requires an ADR update. - -Current sub-sections and their intended scope: - -| Sub-section | Scope | -|---|---| -| Data Formats & Storage | Physical storage formats, encoding trade-offs, cloud-native implications | -| Data Systems & Architecture | Databases, data platforms, pipeline architecture, orchestration | -| Systems Design & Architecture | Service decomposition, distributed systems, scalability trade-offs | -| Operations & Reliability | Observability, monitoring, incident management, operational economics | -| Infrastructure & Automation | Container philosophy, IaC, GitOps, deployment models | -| Embedded & Radio | Microcontroller platforms, IoT protocols, radio systems | - ---- - -## Handling Obsolescence - -Entries that contain analysis that has become materially incorrect due to technology changes should be updated rather than deleted. The update should be noted at the top of the document with the date and nature of the update. If the core analytical claim has been invalidated (not merely that specific tools have changed), the entry should be marked as superseded and a new entry created. - -Entries that are superseded or significantly updated trigger a review of cross-links from other entries to ensure referenced conclusions remain accurate. - ---- - -## Cross-Link Expectations - -Cross-links serve two functions: they help readers navigate the analytical corpus, and they expose the analytical relationships between topics. Entries that are analytically related but not cross-linked create isolated analysis that readers cannot discover through navigation. - -Minimum cross-link expectations: -- Every deep dive links to at least two other entries in the corpus (deep dives, best practices, or tutorials) -- Every new entry is linked from at least one existing entry or from the index -- The deep-dives/index.md is updated for every new entry - -Cross-links are maintained as part of the authoring process, not as a retrospective cleanup task. - ---- - -## Consequences - -This ADR supersedes the inclusion criteria statements in ADRs 0008 and 0009. Those ADRs remain as historical records of the expansion decisions they document; their criteria sections are no longer authoritative. - -Future deep dive entries do not require an ADR. They require adherence to the criteria above. ADR updates are required only for structural changes to the section: new sub-sections, reorganization, or material changes to the governance model itself. - ---- - -## Related Documents - -- [ADR 0009: Deep Dives Expansion](0009-deep-dives-expansion.md) — previous expansion (criteria superseded by this ADR) -- [ADR 0008: Expand Deep Dives Section](0008-expand-deep-dives-section.md) — previous expansion (criteria superseded by this ADR) -- [ADR 0007: Deep Dives Section](0007-deep-dives-section.md) — original section creation decision diff --git a/docs/adr/0011-deep-dives-expansion-phase-2.md b/docs/adr/0011-deep-dives-expansion-phase-2.md deleted file mode 100644 index 421e578..0000000 --- a/docs/adr/0011-deep-dives-expansion-phase-2.md +++ /dev/null @@ -1,90 +0,0 @@ ---- -tags: - - adr - - documentation ---- - -# ADR 0011: Deep Dives Expansion Phase II - -**Status**: Accepted -**Date**: 2026-02-26 -**Author**: Joshua N. Grant - ---- - -## Context - -ADR 0010 established a consolidated governance model for the Deep Dives section and noted that future deep dive entries do not require an ADR — they require adherence to the criteria established in ADR 0010. This ADR documents Phase II of the corpus expansion as a record of scope decisions rather than as a governance update. - -Phase II adds five entries across three new topic clusters: real-time systems economics, data lake governance failure, and distributed systems theory. A fourth entry enhances the existing metadata-as-infrastructure document with a data plane/control plane framework and a metadata debt section. The Human Cost of Automation addresses the organizational side of the infrastructure automation deep dives already present in the corpus. - -The corpus now contains twenty-two entries across six sub-sections. - ---- - -## Entries Added in Phase II - -**Data Systems & Architecture** - -- **The Hidden Cost of Real-Time Systems** — latency taxonomy (hard/soft/streaming/batch), infrastructure costs of stateful stream processing (checkpointing, backpressure, deduplication), observability explosion in streaming, cost comparison table, and a decision framework from batch through micro-batch to streaming -- **Why Most Data Lakes Become Data Swamps** — structural decay mechanisms (missing metadata, schema drift, orphaned datasets, version confusion), technical anti-patterns, cultural failures, lakehouse as partial solution, and zone-based architecture for prevention -- **Metadata as Infrastructure (enhanced)** — addition of data plane vs metadata plane ASCII diagram, metadata debt taxonomy (stale descriptions, broken lineage, inconsistent tags), and updated title framing metadata as a hidden control plane rather than mere documentation - -**Systems Design & Architecture** - -- **Distributed Systems and the Myth of Infinite Scale** — CAP theorem and PACELC analysis, N² coordination growth diagram, data gravity and cloud egress cost, partial failure taxonomy, rolling upgrade complexity, and a decision framework distinguishing genuine distribution requirements from distribution-as-prestige - -**Infrastructure & Automation** - -- **The Human Cost of Automation** — skill atrophy mechanisms, complexity transfer from visible toil to invisible fragility, psychological impact of reduced operator agency, automation maturity model (manual through self-healing), and a decision framework with explicit criteria for when automation creates more organizational risk than it removes - ---- - -## Criteria for Future Inclusion - -ADR 0010 remains the authoritative governance document. For reference, the criteria are: - -**Topic**: genuine architectural tension, consequential scope (1–3 year horizon), non-duplication of existing entries, and analytical durability (not version-specific). - -**Structure**: Opening Thesis → Historical Context → Analytical Core (with comparison and diagram) → Decision Framework → Cross-links. - -**Tone**: analytical (not tutorial, not marketing, not blog voice). Positions must be argued with evidence and logical structure. - ---- - -## Avoiding Intellectual Dilution - -The risk at twenty-two entries is that the corpus begins to include topics that are adjacent to but not clearly within the analytical scope of the section. The test question from ADR 0010 remains the primary filter: would this content belong in an engineering organization's internal knowledge base, cited in architecture review documents? - -Two additional tests apply at scale: - -**The novelty test**: does the proposed entry add an analytical perspective not already present in the corpus? An entry on "Why Most Kubernetes Deployments Are Overcomplicated" that makes the same structural argument as "Why Most Microservices Should Be Monoliths" applied to a different technology does not pass the novelty test. The corpus should provide a range of analytical perspectives, not the same perspective applied to many technology choices. - -**The cross-link test**: a well-positioned deep dive should naturally link to and from at least two existing entries. A proposed entry with no natural cross-links to existing corpus entries is likely analytically isolated from the corpus's themes — a signal that it is not well-suited to the section. - ---- - -## Cross-Linking Discipline - -Cross-links in the Deep Dives corpus serve a function beyond navigation: they reveal the analytical graph structure of the corpus. Entries that form a cluster of mutual cross-links address related analytical questions. The current corpus has two main analytical clusters: - -**Data systems cluster**: Parquet, Lakehouse, DuckDB, Pipelines, Data Swamps, Metadata, Real-Time, Prefect/Airflow — all address different dimensions of the same set of trade-offs in analytical data architecture. - -**Systems and operations cluster**: Microservices, Distributed Systems, Observability, Economics of Observability, Automation, IaC/GitOps, Containers — all address different dimensions of how software systems are built, operated, and governed. - -**Embedded cluster**: LoRaWAN, MQTT, ESP32 — domain-specific analytical content for the IoT/embedded systems audience. - -Cross-links should predominantly connect entries within clusters. Cross-cluster links are appropriate when the analytical connection is genuine (e.g., the Economics of Observability links to Data Pipeline observability) rather than merely topical. - ---- - -## Consequences - -The corpus is mature enough to serve as a coherent analytical reference rather than a collection of independent essays. Future entries should be evaluated for their contribution to the corpus's analytical structure — do they deepen an existing cluster, or do they begin a new cluster with at least two related entries? Isolated single entries on topics without natural corpus neighbors should be evaluated skeptically. - ---- - -## Related Documents - -- [ADR 0010: Governance Model for Deep Dives](0010-deep-dives-governance.md) — authoritative governance (criteria, tone, structure, cross-linking) -- [ADR 0009: Deep Dives Expansion](0009-deep-dives-expansion.md) — Phase I expansion record diff --git a/docs/adr/0012-deep-dives-scale-governance.md b/docs/adr/0012-deep-dives-scale-governance.md deleted file mode 100644 index 47e2ffc..0000000 --- a/docs/adr/0012-deep-dives-scale-governance.md +++ /dev/null @@ -1,138 +0,0 @@ ---- -tags: - - adr - - governance - - deep-dives ---- - -# ADR 0012: Governance Model for Large-Scale Deep Dive Section - -**Status**: Accepted -**Date**: 2026-02-26 -**Supersedes**: Curation criteria in ADR 0010 and ADR 0011 -**Context**: Deep Dives section has grown to 24 essays across 6 thematic clusters. - ---- - -## Context - -The Deep Dives section was established in ADR 0007 to provide a space for long-form analytical essays distinct from tutorials and best practices. It has since expanded through four phases of addition (ADR 0008, 0009, 0010, 0011) and now contains 24 essays spanning data formats, data systems, systems architecture, observability, infrastructure automation, embedded systems, and GPU economics. - -At this scale, the section faces three governance challenges that require explicit policy: - -1. **Curation discipline**: not every analytical question warrants a new deep dive. Without a clear novelty test, the section will accumulate essays that repeat one another, dilute intellectual signal, and create maintenance burden without proportional reader value. - -2. **Cross-linking requirements**: at 24 essays, the potential cross-link surface is large. Without policy, cross-linking becomes arbitrary (linking everything to everything) or stale (links added at creation time but not maintained as the corpus grows). - -3. **Thematic clustering**: the navigation must remain legible as the section grows. Sub-sections that contain too few or too many essays degrade navigation usability. Cluster boundaries must be governed explicitly. - ---- - -## Decision - -### Curation Discipline - -A new deep dive is warranted only when it satisfies all of the following: - -**Novelty test**: the essay addresses a question not adequately covered by any existing deep dive, even by implication. If the answer to "does this already exist?" is "mostly, in another essay," the appropriate action is to add a section to the existing essay, not create a new file. - -**Analytical substance test**: the essay must contain genuine comparative analysis — trade-off matrices, historical context, ecosystem maturity commentary, economic implications — not just an informed opinion or a list of considerations. If it cannot sustain 800+ words of structured argument, it is not a deep dive. - -**Durability test**: the essay must address a structural question with multi-year relevance. Technology announcements, version-specific comparisons, and tactical "how to choose" guides are not deep dives. A deep dive on Kubernetes overhead remains relevant as Kubernetes itself evolves; an essay on "Kubernetes 1.29 upgrade tips" does not. - -**Non-tutorial test**: the essay must not contain step-by-step implementation instructions. If it does, the implementation content belongs in the Tutorials section and the analytical content belongs in Deep Dives or Best Practices. - -### Cross-Linking Requirements - -Every new deep dive must include at minimum 2 and at maximum 5 internal cross-links to other deep dives, placed at the opening (header note) and/or closing (See also admonition). Cross-links must be directionally meaningful — they should explain *why* the linked essay is relevant, not merely that it exists. - -When a new deep dive is added, at minimum 2 existing essays that are conceptually adjacent must be updated to add a reciprocal cross-link. This maintains the web of connections as the corpus grows and prevents new essays from being isolated. - -Cross-links to Best Practices and Tutorial pages are appropriate when the deep dive directly extends or contextualizes a practice or tutorial. These links should appear in the See also admonition only. - -### Thematic Clustering - -The six current thematic clusters are: - -| Cluster | Current count | Soft maximum | -|---|---|---| -| Data Formats & Storage | 3 | 6 | -| Data Systems & Architecture | 8 | 12 | -| Systems Design & Architecture | 4 | 7 | -| Operations & Reliability | 2 | 5 | -| Infrastructure & Automation | 4 | 7 | -| Embedded & Radio | 3 | 6 | - -When a cluster approaches its soft maximum, the appropriate response is one of: -- Split the cluster into two sub-clusters with distinct thematic identity -- Merge essays that substantially overlap into a revised composite essay -- Enforce the novelty test more strictly for proposed additions to that cluster - -Sub-cluster maximum: 12 essays per thematic cluster in the navigation. Beyond 12, navigation usability degrades to the point where sub-cluster splitting is mandatory. - -### Obsolescence Handling - -Deep dives become partially obsolete when the ecosystem evolves significantly: a technology ceases to be relevant, a comparison changes due to product acquisition or open-source abandonment, or the fundamental trade-off the essay describes resolves in one direction definitively. The response is: - -- **Minor obsolescence** (one section is stale): add a dated note at the beginning of the affected section. Do not rewrite the essay. -- **Major obsolescence** (the central thesis is invalidated): mark the essay with a deprecation notice at the top, link to any successor essay, and leave the original content in place for historical context. -- **Supersession** (a new essay covers the same territory with superior analysis): the new essay should explicitly state what it supersedes; the old essay should link to the new one. - ---- - -## Thematic Cluster Map (Current State) - -At the time of this ADR: - -``` -Deep Dives (24 essays) -│ -├── Data Formats & Storage (3) -│ Parquet · Geospatial Formats · Polars vs Pandas Geospatial -│ -├── Data Systems & Architecture (8) -│ Lakehouse · DuckDB vs PG vs Spark · Data Pipeline Failures · -│ Data Lakes → Swamps · Prefect vs Airflow · Metadata as Infra · -│ Hidden Cost of Real-Time · End of the Data Warehouse? -│ -├── Systems Design & Architecture (4) -│ Monolith vs Microservices · Appropriate Microservices · -│ Distributed Scale Myths · Kubernetes Overhead -│ -├── Operations & Reliability (2) -│ Observability vs Monitoring · Economics of Observability -│ -├── Infrastructure & Automation (4) -│ Container Base Images · IaC vs GitOps · -│ Human Cost of Automation · Economics of GPU Infrastructure -│ -└── Embedded & Radio (3) - LoRaWAN vs Raw LoRa · MQTT vs HTTP · ESP32 vs Raspberry Pi -``` - ---- - -## Rationale - -The governance model prioritizes intellectual coherence over volume. A corpus of 24 high-quality essays is more valuable than a corpus of 50 essays of inconsistent depth. The soft cluster maximums create a forcing function: adding a new essay to a near-full cluster requires either demonstrating the novelty test is met or splitting the cluster, both of which impose a productive discipline on scope. - -The cross-linking requirement prevents the section from fragmenting into isolated essays. The reciprocal link requirement is the mechanism that keeps the web of connections current as the corpus grows. - ---- - -## Alternatives Considered - -**No explicit governance**: allow the section to grow organically, with ADRs created only for major structural changes. Rejected because at 24 essays, without explicit policy, the section will accumulate redundant essays and navigation will degrade. - -**Hard essay limits**: set an absolute maximum (e.g., 30 essays). Rejected because the correct limit is topical coverage, not count. A section with 25 essays each addressing a distinct structural question is better than 20 essays with significant overlap. - -**Separate site section for each thematic cluster**: treat Data Systems deep dives as a separate section from Systems Design deep dives. Rejected because the cross-cluster linkage is a feature — data system decisions interact with systems design decisions — and the unified Deep Dives section preserves navigational context for that interaction. - ---- - -## Consequences - -- Future deep dives must pass the novelty, substance, durability, and non-tutorial tests before creation. -- New essays must include 2–5 internal cross-links and trigger reciprocal cross-link updates in at least 2 existing essays. -- Navigation sub-cluster sizes must be monitored; splitting is mandatory when a cluster exceeds 12 entries. -- Obsolescence handling policy is now explicit and must be applied when significant ecosystem changes render existing essays partially stale. diff --git a/docs/adr/0013-deep-dives-curation-model.md b/docs/adr/0013-deep-dives-curation-model.md deleted file mode 100644 index 39a3751..0000000 --- a/docs/adr/0013-deep-dives-curation-model.md +++ /dev/null @@ -1,112 +0,0 @@ ---- -tags: - - adr - - governance - - deep-dives ---- - -# ADR 0013: Deep Dive Curation and Thematic Clustering Model - -**Status**: Accepted -**Date**: 2026-02-26 -**Supersedes**: Curation criteria in ADR 0012 (governance thresholds now updated) -**Context**: Deep Dives section has grown to 29 essays across 6 thematic clusters. - ---- - -## Context - -ADR 0012 established a governance model for the Deep Dives section when it contained 24 essays. This ADR refines that model as the corpus has grown to 29 essays — crossing the ADR 0012 soft cluster maximum for two clusters (Data Formats & Storage: 5 entries; Data Systems & Architecture: 10 entries) — and as experience with the cross-linking and curation requirements has revealed patterns that require more explicit guidance. - -The primary governance challenge at this scale is **conceptual drift**: essays that individually satisfy the novelty and substance tests but that, in aggregate, produce a corpus whose thematic coherence is harder for readers to perceive. A corpus of 29 essays organized into 6 clusters is legible only if the cluster boundaries are sharp enough that readers can use them to navigate rather than reading the entire index. - -The secondary challenge is **cross-link maintenance**: at 29 essays, the potential cross-link surface is large enough that maintaining all cross-links manually as essays are added requires disciplined tracking. The current practice of adding cross-links only at creation time and updating 2–5 existing pages produces a cross-link graph that grows slower than the corpus. - ---- - -## Decision - -### Curation: Thematic Fit as Prerequisite - -Every proposed deep dive must be assignable to exactly one existing thematic cluster before creation. If an essay spans multiple clusters with equal weight, it either belongs in the cluster where its *primary argument* lives, or it reveals a missing cluster boundary. Essays that cannot be assigned to a cluster without forcing are evidence of scope problem rather than thematic novelty. - -The curation question is: **"what cluster would a reader look in to find this essay?"** If the answer requires describing a cluster that does not exist, the question becomes: "is there sufficient topical density to justify a new cluster, or is the essay an edge case?" - -### Cluster Definitions (Updated) - -The six clusters are now explicitly defined by their organizing analytical question: - -| Cluster | Organizing question | Current count | -|---|---|---| -| Data Formats & Storage | What physical and format-level constraints govern how data is stored and accessed? | 5 | -| Data Systems & Architecture | What architectural and organizational forces shape analytical data systems? | 10 | -| Systems Design & Architecture | What structural limits constrain the design of distributed and decomposed software systems? | 4 | -| Operations & Reliability | What operational and economic forces govern system observability, monitoring, and reliability? | 2 | -| Infrastructure & Automation | What organizational and technical trade-offs govern infrastructure acquisition, automation, and compute economics? | 5 | -| Embedded & Radio | What physical and ecosystem constraints govern embedded system and radio protocol selection? | 3 | - -**Data Systems & Architecture** has reached the soft maximum of 10. The next addition to this cluster requires either demonstrating that the proposed essay has a sufficiently distinct primary argument from the 10 existing essays, or splitting the cluster into two sub-clusters. Candidate split: "Data Storage & Table Formats" (Lakehouse, Warehouse, Parquet, Metadata, Data Lakes, Data Pipelines) vs "Data Computation & Orchestration" (DuckDB/PG/Spark, Prefect/Airflow, Real-Time, ML Systems). - -### Cross-Linking: Minimum Requirements - -Every new essay must: - -1. Include a header cross-link (`*See also: ...*`) to 1–2 essays in *other* clusters, explaining why the reader should continue there. -2. Include a closing `!!! tip "See also"` admonition with 3–5 cross-links with directionally meaningful descriptions. -3. Trigger reciprocal updates in at least 2 existing essays that are the most thematically adjacent. - -The purpose of cross-cluster header links is navigational: a reader who arrived at a Data Systems essay from the index should have a visible path to the Infrastructure cluster if the essay assumes infrastructure concepts. The purpose of closing cross-links is analytical: they tell the reader what questions to pursue after this essay. - -### Avoiding Conceptual Drift - -The symptom of conceptual drift is when a reader looking at the index cannot determine, from the title and one-sentence description, what distinguishes one essay from three adjacent essays. The prevention mechanism is the **distinction test**: before creating a new essay, identify the 2–3 most topically similar existing essays and articulate, in one sentence, how the new essay's central argument differs from each. If the distinction cannot be articulated clearly, the essay is not ready for creation. - ---- - -## Current Corpus Map (29 essays) - -``` -Deep Dives (29 essays) -│ -├── Data Formats & Storage (5) -│ Parquet · Geospatial Formats · Polars vs Pandas Geospatial · -│ Physics of Storage Systems · Operational Geometry of Spatial Systems -│ -├── Data Systems & Architecture (10) ← approaching split threshold -│ Lakehouse · DuckDB vs PG vs Spark · Data Pipeline Failures · -│ Data Lakes → Swamps · Prefect vs Airflow · Metadata as Infra · -│ Hidden Cost of Real-Time · End of the Data Warehouse? · -│ Hidden Cost of Metadata Debt · Why ML Systems Fail in Production -│ -├── Systems Design & Architecture (4) -│ Monolith vs Microservices · Appropriate Microservices · -│ Distributed Scale Myths · Kubernetes Overhead -│ -├── Operations & Reliability (2) -│ Observability vs Monitoring · Economics of Observability -│ -├── Infrastructure & Automation (5) -│ Container Base Images · IaC vs GitOps · -│ Human Cost of Automation · Economics of GPU Infrastructure · -│ Myth of Serverless Simplicity -│ -└── Embedded & Radio (3) - LoRaWAN vs Raw LoRa · MQTT vs HTTP · ESP32 vs Raspberry Pi -``` - ---- - -## Rationale - -The governance model deliberately imposes friction on new additions. The cost of this friction is slower corpus growth. The benefit is a corpus whose individual essays are more distinct from one another, whose cluster boundaries are more navigable, and whose cross-link graph is maintained rather than stale. The section is more valuable at 30 high-distinction essays than at 50 essays with significant conceptual overlap. - -The cluster split decision for Data Systems & Architecture is deferred to the next addition; the split should be made based on where the 11th essay falls, not in anticipation of a hypothetical essay. - ---- - -## Consequences - -- Proposed essays must satisfy the distinction test (articulated difference from 2–3 most similar existing essays) before creation. -- Data Systems & Architecture approaching 10 essays requires monitoring; the 11th addition triggers a cluster split decision. -- Cross-linking minimums (header link + 3–5 closing links + 2 reciprocal updates) are formalized as creation requirements. -- ADR 0012 curation criteria remain valid; this ADR refines them for a larger corpus and adds the distinction test and cluster definition model. diff --git a/docs/adr/0014-elevate-site-to-systems-doctrine.md b/docs/adr/0014-elevate-site-to-systems-doctrine.md deleted file mode 100644 index a63c966..0000000 --- a/docs/adr/0014-elevate-site-to-systems-doctrine.md +++ /dev/null @@ -1,119 +0,0 @@ -# ADR 0014: Elevate Site from Documentation Collection to Cohesive Systems Doctrine - -- Status: Accepted -- Date: 2026-02-26 -- Deciders: Joshua Grant -- Technical Story: Transition the site from a growing collection of tutorials and analytical essays into a coherent, navigable systems-level doctrine. - ---- - -## Context - -The site has evolved beyond tutorials and best practices into a substantial body of analytical deep dives covering: - -- Distributed systems -- Storage theory -- Observability -- Data architecture -- Embedded systems -- Infrastructure economics -- Geospatial systems - -While the intellectual depth has increased, structural cohesion has not kept pace. - -The risks are: - -- Content silos -- Cognitive overload for new readers -- Redundant arguments across deep dives -- Lack of narrative entry points -- Weak conceptual mapping between sections - -The site is at an inflection point: -It can either become a loosely connected archive, or a structured intellectual framework. - ---- - -## Decision - -We will introduce structural layers that transform the site into a cohesive systems doctrine: - -1. **Architectural Entry Points** - - A “Start Here” decision tree page - - Curated Reading Tracks - -2. **Thematic Cohesion** - - Cross-cutting theme declarations - - A Decision Framework Index - - An Anti-Patterns Index - -3. **Intellectual Framing** - - A “Philosophy of the Site” page - - A Systems Thinking Glossary - - A Canon / Recommended Reading order - -4. **Cross-Section Synthesis** - - Meta-essays connecting deep dives - - Explicit cross-links between related doctrines - -5. **Intentional Structure** - - Minimal redundancy - - Clear semantic relationships - - Designed, not accumulated navigation - ---- - -## Alternatives Considered - -### 1. Continue Adding Content Without Structural Reform -Rejected. -This risks fragmentation and reader fatigue. - -### 2. Create an External Blog for Philosophy -Rejected. -The philosophy is intrinsic to the technical content and should remain integrated. - -### 3. Major Structural Rewrite -Rejected. -We prefer incremental, compositional refinement. - ---- - -## Consequences - -### Positive - -- Increased intellectual cohesion -- Clear onboarding path for new readers -- Stronger narrative continuity -- Reduced conceptual redundancy -- Elevated authority and intentionality - -### Negative - -- Additional structural overhead -- Increased editorial discipline required -- Slightly more complex navigation - ---- - -## Follow-ups - -- Implement "Start Here" decision tree -- Add Reading Tracks page -- Add Decision Framework Index -- Add Anti-Patterns index -- Add Philosophy page -- Add Systems Glossary -- Introduce light thematic labeling - ---- - -## Governance - -Future deep dives must: - -- Declare themes -- Cross-link to at least two existing essays -- Avoid duplication of core arguments -- End with a structured decision framework \ No newline at end of file diff --git a/docs/adr/0015-diagrams-mermaid.md b/docs/adr/0015-diagrams-mermaid.md deleted file mode 100644 index 1ec0ef1..0000000 --- a/docs/adr/0015-diagrams-mermaid.md +++ /dev/null @@ -1,93 +0,0 @@ ---- -tags: - - adr - - diagrams - - documentation ---- - -# ADR 0015: Standardize Site Diagrams on Mermaid (with SVG as Optional Exception) - -**Status**: Accepted -**Date**: 2026-02-27 -**Deciders**: Joshua N. Grant -**Technical Story**: Replace ad-hoc ASCII diagrams with a consistent, maintainable diagram standard that is version-friendly and renders cleanly across the site. - ---- - -## Context - -The site increasingly relies on conceptual diagrams to explain architecture, trade-offs, and system boundaries. Many diagrams are currently written as ASCII. While ASCII is portable and fast to write, it has several drawbacks: - -- Limited expressive power for complex relationships -- Inconsistent visual vocabulary across pages -- Reduced readability on mobile and narrow screens -- Difficult to reuse or standardize diagram patterns -- Perceived "sketchy" quality even when content is rigorous - -SVG diagrams can look excellent but tend to introduce tooling overhead: - -- Requires authoring tools or generation pipelines -- Harder to edit and review as text diffs -- Risk of inconsistent styling unless a strict convention exists - -Mermaid offers a middle ground: - -- Text-based diagrams (diff-friendly, reviewable) -- Easy to embed in Markdown -- Supports common diagram types (flowcharts, sequence diagrams, state diagrams) -- Low friction for authors -- Rendered by MkDocs Material through `pymdownx.superfences` — already configured - ---- - -## Decision - -We will standardize on **Mermaid** for the majority of diagrams across the site. - -- Mermaid becomes the default for architecture, flow, state, and sequence diagrams. -- SVG remains an optional exception for: - - High-resolution posters and figures - - Geospatial illustrations and maps - - Cases where Mermaid is insufficient or visually unclear - -We will: - -1. Confirm Mermaid rendering is enabled in `mkdocs.yml` (it is — via `pymdownx.superfences` custom fence). -2. Add a diagram style guide at `docs/diagrams/style-guide.md`. -3. Convert a starter set of high-value ASCII diagrams to Mermaid (see Follow-ups). -4. Gradually convert remaining ASCII diagrams during normal content evolution. - ---- - -## Alternatives Considered - -**Continue using ASCII diagrams**: Rejected. Lacks consistency and visual clarity at scale. - -**Adopt SVG as the default**: Rejected. Higher authoring friction and tooling requirements. - -**Use both equally without a default**: Rejected. Without a default standard, style fragments and diagrams drift. - ---- - -## Consequences - -**Positive**: -- Higher clarity and visual consistency across the site -- Better mobile readability -- Text-based diffs for diagrams in pull requests -- Reduced diagram authoring friction compared to SVG -- Stronger perceived polish without sacrificing maintainability - -**Negative / Trade-offs**: -- Mermaid features vary by diagram type and renderer version -- Some complex diagrams may require careful layout constraints -- Small learning curve for contributors - ---- - -## Follow-ups - -- [x] Confirm `pymdownx.superfences` mermaid custom fence is in `mkdocs.yml` -- [x] Add `docs/diagrams/style-guide.md` with snippet library -- [x] Convert core doctrine diagrams: Prefect vs Airflow, ESP32 vs Pi, Data Pipelines, Observability vs Monitoring -- [ ] Convert remaining ASCII diagrams gradually during content evolution diff --git a/docs/adr/_templates/0001-template-standard.md b/docs/adr/_templates/0001-template-standard.md deleted file mode 100644 index ab1e9df..0000000 --- a/docs/adr/_templates/0001-template-standard.md +++ /dev/null @@ -1,81 +0,0 @@ -# ADR-XXXX: [Short Descriptive Title] - -**Status**: Proposed | Accepted | Superseded | Rejected | Deprecated - -**Date**: YYYY-MM-DD - -**Deciders**: [Names or roles, e.g., "Architecture Working Group", "Tech Leads"] - -**Tags**: [Optional: component, domain, e.g., "observability", "security", "data"] - ---- - -## Context - -[2-4 paragraphs describing the situation, problem, or requirement that led to this decision. Include: -- What problem are we solving? -- What constraints exist? -- What are the business/technical drivers?] - -## Decision - -[1-2 paragraphs stating the decision clearly and concisely. Be specific about what is being decided.] - -We will [specific action/choice]. - -## Options Considered - -### Option 1: [Name of Option] - -**Description**: [What this option entails] - -**Pros**: -- [Benefit 1] -- [Benefit 2] - -**Cons**: -- [Drawback 1] -- [Drawback 2] - -### Option 2: [Name of Option] - -[Same structure as Option 1] - -### Option 3: [Name of Option] - -[Same structure as Option 1] - -## Rationale - -[2-3 paragraphs explaining why the chosen option was selected. Reference specific pros/cons, constraints, or requirements that drove the decision.] - -## Consequences - -### Positive - -- [Expected positive outcome 1] -- [Expected positive outcome 2] - -### Negative - -- [Expected negative outcome 1] -- [Expected negative outcome 2] - -### Neutral / Trade-offs - -- [Trade-off or neutral consequence] - -## Related Documents - -- [Link to design doc, RFC, or implementation guide] -- [Link to best-practices document] -- [Link to runbook or operational guide] - -## Implementation Notes - -[Optional: Brief notes on how this decision will be implemented, key milestones, or dependencies] - -## References - -[Optional: Links to external resources, research, or benchmarks that informed the decision] - diff --git a/docs/adr/_templates/0002-template-lightweight.md b/docs/adr/_templates/0002-template-lightweight.md deleted file mode 100644 index 96f58d5..0000000 --- a/docs/adr/_templates/0002-template-lightweight.md +++ /dev/null @@ -1,28 +0,0 @@ -# ADR-XXXX: [Short Descriptive Title] - -**Status**: Proposed | Accepted | Superseded | Rejected | Deprecated - -**Date**: YYYY-MM-DD - -**Deciders**: [Names or roles] - ---- - -## Context - -[1-2 paragraphs: What problem or situation led to this decision?] - -## Decision - -[1 paragraph: What is being decided?] - -We will [specific action/choice]. - -## Impact / Notes - -[1-2 paragraphs: What are the key consequences, trade-offs, or implementation considerations?] - -## Related Documents - -- [Link to relevant docs] - diff --git a/docs/adr/index.md b/docs/adr/index.md deleted file mode 100644 index 4c6cbc0..0000000 --- a/docs/adr/index.md +++ /dev/null @@ -1,30 +0,0 @@ -# Architecture Decision Records - -Architecture Decision Records (ADRs) document significant decisions made while designing and building this site — what was chosen, what was rejected, and why. - -See [ADR and Technical Decision Governance](../best-practices/architecture-design/adr-decision-governance.md) for the methodology behind these records. - -## ADR Index - -| Number | Title | Status | Date | -|--------|-------|--------|------| -| [ADR-0001](0001-site-architecture.md) | Site Architecture — MkDocs + Material + Repo-as-Docs | Accepted | 2026-02-28 | -| [ADR-0002](0002-why-mkdocs-material.md) | Use MkDocs + Material Theme | Accepted | 2024-01-15 | -| [ADR-0003](0003-why-best-practices-vs-tutorials.md) | Separate Best Practices from Tutorials | Accepted | 2024-02-01 | -| [ADR-0004](0004-why-just-for-fun-exists.md) | Include a Just for Fun Section | Accepted | 2024-03-10 | -| [ADR-0005](0005-esp32-section-architecture.md) | Establish ESP32 & Embedded Systems as a First-Class Section | Accepted | 2026-02-26 | -| [ADR-0006](0006-embedded-expansion-lora-power.md) | Expand Embedded Section — LoRa, Power Electronics, Variant Notes | Accepted | 2026-02-26 | -| [ADR-0007](0007-deep-dives-section.md) | Introduce "Deep Dives" Section for Analytical Long-Form Content | Accepted | 2026-02-26 | -| [ADR-0008](0008-expand-deep-dives-section.md) | Expand Deep Dives as a First-Class Analytical Section | Accepted | 2026-02-26 | -| [ADR-0009](0009-deep-dives-expansion.md) | Expand Deep Dives into Core Analytical Layer | Accepted | 2026-02-26 | -| [ADR-0010](0010-deep-dives-governance.md) | Governance Model for Deep Dives | Accepted | 2026-02-26 | -| [ADR-0011](0011-deep-dives-expansion-phase-2.md) | Deep Dives Expansion Phase II | Accepted | 2026-02-26 | -| [ADR-0012](0012-deep-dives-scale-governance.md) | Governance Model for Large-Scale Deep Dive Section | Accepted | 2026-02-26 | -| [ADR-0013](0013-deep-dives-curation-model.md) | Deep Dive Curation and Thematic Clustering Model | Accepted | 2026-02-26 | -| [ADR-0014](0014-elevate-site-to-systems-doctrine.md) | Elevate Site from Documentation Collection to Cohesive Systems Doctrine | Accepted | 2026-02-26 | -| [ADR-0015](0015-diagrams-mermaid.md) | Standardize Site Diagrams on Mermaid | Accepted | 2026-02-27 | - -## Templates - -- **[Standard ADR Template](_templates/0001-template-standard.md)** — For most architectural decisions -- **[Lightweight ADR Template](_templates/0002-template-lightweight.md)** — For smaller, focused decisions diff --git a/docs/tutorials/best-practices-integration/api-first-polyglot-protobuf.md b/docs/archive/tutorials/best-practices-integration/api-first-polyglot-protobuf.md similarity index 100% rename from docs/tutorials/best-practices-integration/api-first-polyglot-protobuf.md rename to docs/archive/tutorials/best-practices-integration/api-first-polyglot-protobuf.md diff --git a/docs/tutorials/best-practices-integration/cost-optimized-geokg-capacity-planning.md b/docs/archive/tutorials/best-practices-integration/cost-optimized-geokg-capacity-planning.md similarity index 100% rename from docs/tutorials/best-practices-integration/cost-optimized-geokg-capacity-planning.md rename to docs/archive/tutorials/best-practices-integration/cost-optimized-geokg-capacity-planning.md diff --git a/docs/tutorials/best-practices-integration/cost-optimized-ml-pipeline-capacity-retention.md b/docs/archive/tutorials/best-practices-integration/cost-optimized-ml-pipeline-capacity-retention.md similarity index 100% rename from docs/tutorials/best-practices-integration/cost-optimized-ml-pipeline-capacity-retention.md rename to docs/archive/tutorials/best-practices-integration/cost-optimized-ml-pipeline-capacity-retention.md diff --git a/docs/tutorials/best-practices-integration/data-pipeline-quality-governance-observability.md b/docs/archive/tutorials/best-practices-integration/data-pipeline-quality-governance-observability.md similarity index 100% rename from docs/tutorials/best-practices-integration/data-pipeline-quality-governance-observability.md rename to docs/archive/tutorials/best-practices-integration/data-pipeline-quality-governance-observability.md diff --git a/docs/tutorials/best-practices-integration/developer-experience-taxonomy-repository.md b/docs/archive/tutorials/best-practices-integration/developer-experience-taxonomy-repository.md similarity index 100% rename from docs/tutorials/best-practices-integration/developer-experience-taxonomy-repository.md rename to docs/archive/tutorials/best-practices-integration/developer-experience-taxonomy-repository.md diff --git a/docs/tutorials/best-practices-integration/documentation-driven-adr-governance.md b/docs/archive/tutorials/best-practices-integration/documentation-driven-adr-governance.md similarity index 99% rename from docs/tutorials/best-practices-integration/documentation-driven-adr-governance.md rename to docs/archive/tutorials/best-practices-integration/documentation-driven-adr-governance.md index 378b88d..99897b4 100644 --- a/docs/tutorials/best-practices-integration/documentation-driven-adr-governance.md +++ b/docs/archive/tutorials/best-practices-integration/documentation-driven-adr-governance.md @@ -4,7 +4,7 @@ This tutorial combines: - **[Documentation Best Practices](../../best-practices/architecture-design/documentation.md)** - Writing and organizing useful documentation -- **[ADR and Technical Decision Governance](../../best-practices/architecture-design/adr-decision-governance.md)** - Architecture Decision Records +- **[Architecture & Design documentation](../../best-practices/architecture-design/documentation.md)** - includes ADR patterns as a general practice - **[Reference Architecture Diagrams](../../best-practices/architecture-design/reference-architecture-diagrams.md)** - System documentation - **[Cognitive Load Management and Developer Experience](../../best-practices/architecture-design/cognitive-load-developer-experience.md)** - Developer experience optimization diff --git a/docs/tutorials/best-practices-integration/event-driven-geokg-observability.md b/docs/archive/tutorials/best-practices-integration/event-driven-geokg-observability.md similarity index 100% rename from docs/tutorials/best-practices-integration/event-driven-geokg-observability.md rename to docs/archive/tutorials/best-practices-integration/event-driven-geokg-observability.md diff --git a/docs/tutorials/best-practices-integration/event-driven-microservices-observability-data-governance.md b/docs/archive/tutorials/best-practices-integration/event-driven-microservices-observability-data-governance.md similarity index 100% rename from docs/tutorials/best-practices-integration/event-driven-microservices-observability-data-governance.md rename to docs/archive/tutorials/best-practices-integration/event-driven-microservices-observability-data-governance.md diff --git a/docs/tutorials/best-practices-integration/geospatial-data-mesh-cost-capacity.md b/docs/archive/tutorials/best-practices-integration/geospatial-data-mesh-cost-capacity.md similarity index 100% rename from docs/tutorials/best-practices-integration/geospatial-data-mesh-cost-capacity.md rename to docs/archive/tutorials/best-practices-integration/geospatial-data-mesh-cost-capacity.md diff --git a/docs/tutorials/best-practices-integration/high-performance-caching-topology.md b/docs/archive/tutorials/best-practices-integration/high-performance-caching-topology.md similarity index 100% rename from docs/tutorials/best-practices-integration/high-performance-caching-topology.md rename to docs/archive/tutorials/best-practices-integration/high-performance-caching-topology.md diff --git a/docs/tutorials/best-practices-integration/high-performance-geokg-async-caching.md b/docs/archive/tutorials/best-practices-integration/high-performance-geokg-async-caching.md similarity index 100% rename from docs/tutorials/best-practices-integration/high-performance-geokg-async-caching.md rename to docs/archive/tutorials/best-practices-integration/high-performance-geokg-async-caching.md diff --git a/docs/archive/tutorials/best-practices-integration/index.md b/docs/archive/tutorials/best-practices-integration/index.md new file mode 100644 index 0000000..2d0f459 --- /dev/null +++ b/docs/archive/tutorials/best-practices-integration/index.md @@ -0,0 +1,10 @@ +# Archived: Best Practices Integration tutorials + +These walkthroughs combined many topics into single “full stack” narratives. They read like generated course material and are **not maintained** as part of the public tutorial set. + +Use instead: + +- [Tutorials index](../../../tutorials/index.md) +- [Best Practices index](../../../best-practices/index.md) + +Individual archived files remain in this directory for reference during cleanup. diff --git a/docs/tutorials/best-practices-integration/multi-environment-deployment-secrets-config-governance.md b/docs/archive/tutorials/best-practices-integration/multi-environment-deployment-secrets-config-governance.md similarity index 100% rename from docs/tutorials/best-practices-integration/multi-environment-deployment-secrets-config-governance.md rename to docs/archive/tutorials/best-practices-integration/multi-environment-deployment-secrets-config-governance.md diff --git a/docs/tutorials/best-practices-integration/multi-region-dr-temporal-blast-radius.md b/docs/archive/tutorials/best-practices-integration/multi-region-dr-temporal-blast-radius.md similarity index 100% rename from docs/tutorials/best-practices-integration/multi-region-dr-temporal-blast-radius.md rename to docs/archive/tutorials/best-practices-integration/multi-region-dr-temporal-blast-radius.md diff --git a/docs/tutorials/best-practices-integration/multi-region-geokg-temporal-dr.md b/docs/archive/tutorials/best-practices-integration/multi-region-geokg-temporal-dr.md similarity index 100% rename from docs/tutorials/best-practices-integration/multi-region-geokg-temporal-dr.md rename to docs/archive/tutorials/best-practices-integration/multi-region-geokg-temporal-dr.md diff --git a/docs/tutorials/best-practices-integration/polyglot-streaming-saga-cqrs-multicloud.md b/docs/archive/tutorials/best-practices-integration/polyglot-streaming-saga-cqrs-multicloud.md similarity index 100% rename from docs/tutorials/best-practices-integration/polyglot-streaming-saga-cqrs-multicloud.md rename to docs/archive/tutorials/best-practices-integration/polyglot-streaming-saga-cqrs-multicloud.md diff --git a/docs/tutorials/best-practices-integration/progressive-delivery-release-fitness.md b/docs/archive/tutorials/best-practices-integration/progressive-delivery-release-fitness.md similarity index 100% rename from docs/tutorials/best-practices-integration/progressive-delivery-release-fitness.md rename to docs/archive/tutorials/best-practices-integration/progressive-delivery-release-fitness.md diff --git a/docs/tutorials/best-practices-integration/quality-governed-geokg-retention-lifecycle.md b/docs/archive/tutorials/best-practices-integration/quality-governed-geokg-retention-lifecycle.md similarity index 100% rename from docs/tutorials/best-practices-integration/quality-governed-geokg-retention-lifecycle.md rename to docs/archive/tutorials/best-practices-integration/quality-governed-geokg-retention-lifecycle.md diff --git a/docs/tutorials/best-practices-integration/resilient-microservices-chaos-engineering.md b/docs/archive/tutorials/best-practices-integration/resilient-microservices-chaos-engineering.md similarity index 100% rename from docs/tutorials/best-practices-integration/resilient-microservices-chaos-engineering.md rename to docs/archive/tutorials/best-practices-integration/resilient-microservices-chaos-engineering.md diff --git a/docs/tutorials/best-practices-integration/secure-polyglot-identity-encryption.md b/docs/archive/tutorials/best-practices-integration/secure-polyglot-identity-encryption.md similarity index 100% rename from docs/tutorials/best-practices-integration/secure-polyglot-identity-encryption.md rename to docs/archive/tutorials/best-practices-integration/secure-polyglot-identity-encryption.md diff --git a/docs/tutorials/best-practices-integration/semantic-knowledge-graph-rdf-lineage.md b/docs/archive/tutorials/best-practices-integration/semantic-knowledge-graph-rdf-lineage.md similarity index 100% rename from docs/tutorials/best-practices-integration/semantic-knowledge-graph-rdf-lineage.md rename to docs/archive/tutorials/best-practices-integration/semantic-knowledge-graph-rdf-lineage.md diff --git a/docs/assets/overrides/main.html b/docs/assets/overrides/main.html index 524d44b..8bbc215 100644 --- a/docs/assets/overrides/main.html +++ b/docs/assets/overrides/main.html @@ -1,26 +1,7 @@ {% extends "base.html" %} {% block extrahead %} - - - - - - - - - - - - - - - - - - - - - - -{% endblock %} - -{% block content %} - {{ super() }} - - - {% endblock %} diff --git a/docs/assets/overrides/partials/footer.html b/docs/assets/overrides/partials/footer.html deleted file mode 100644 index 99eacac..0000000 --- a/docs/assets/overrides/partials/footer.html +++ /dev/null @@ -1,57 +0,0 @@ - - diff --git a/docs/assets/overrides/partials/header.html b/docs/assets/overrides/partials/header.html deleted file mode 100644 index efd931f..0000000 --- a/docs/assets/overrides/partials/header.html +++ /dev/null @@ -1,85 +0,0 @@ - -
- -
diff --git a/docs/assets/styles/custom.css b/docs/assets/styles/custom.css index 050c0f2..5c8692b 100644 --- a/docs/assets/styles/custom.css +++ b/docs/assets/styles/custom.css @@ -252,17 +252,30 @@ margin: 1.5em 0; } -.project-card { - background: var(--md-default-bg-color); - border: 1px solid var(--md-default-fg-color--lightest); +[data-md-color-scheme="default"] { + --project-card-bg: hsl(0, 0%, 100%); + --project-card-fg: hsla(0, 0%, 0%, 0.87); + --project-card-border: hsla(0, 0%, 0%, 0.12); +} + +[data-md-color-scheme="slate"] { + --project-card-bg: hsl(232, 7%, 16%); + --project-card-fg: hsla(0, 0%, 100%, 0.87); + --project-card-border: hsla(0, 0%, 100%, 0.12); +} + +.md-typeset .project-card { + background: var(--project-card-bg); + color: var(--project-card-fg); + border: 1px solid var(--project-card-border); border-radius: 8px; - padding: 1.25em 1.35em; - box-shadow: 0 1px 3px rgba(0, 0, 0, 0.05); + padding: 1em 1.15em; + box-shadow: none; } .project-card h3 { margin-top: 0; - font-size: 1.15em; + font-size: 1.1em; color: var(--md-primary-fg-color); } @@ -274,9 +287,9 @@ .project-card__image { display: block; width: 100%; - max-width: 140px; + max-width: 120px; height: auto; - margin: 0 auto 0.75em; + margin: 0 auto 0.5em; border-radius: 6px; } diff --git a/docs/best-practices/architecture-design/adr-decision-governance.md b/docs/best-practices/architecture-design/adr-decision-governance.md deleted file mode 100644 index 1510a67..0000000 --- a/docs/best-practices/architecture-design/adr-decision-governance.md +++ /dev/null @@ -1,1599 +0,0 @@ -# Best Practices for Architecture Decision Records (ADRs) and Technical Decision Governance - -**Objective**: Establish a consistent, discoverable practice for recording and governing architecture and technical decisions. When you need to understand why a system is built a certain way, when you want to make decisions explicit and reviewable, when you need to prevent architectural amnesia—this guide provides the framework. - -## Goals & Non-Goals - -### Goals - -- **Create a consistent practice** for recording architecture and technical decisions -- **Make decisions discoverable, reviewable, and auditable** over time -- **Integrate ADRs into existing Git-based workflows** (PRs, issues, reviews) -- **Reduce tribal knowledge** and forgotten decisions -- **Enable better onboarding** by documenting "why" alongside "how" - -### Non-Goals - -- **Not replacing detailed design docs or RFCs**: ADRs capture decisions; design docs capture implementation details -- **Not mandating a single ADR tooling stack**: Focus on patterns and structure that work with any tooling -- **Not requiring ADRs for every change**: Only significant architectural decisions -- **Not a replacement for code review**: ADRs complement, don't replace, code review - -## Why ADRs Matter (Especially in This Kind of Repo) - -### The Problem: Architectural Amnesia - -Your repository already contains extensive best-practices documentation: -- Docker and containerization patterns -- PostgreSQL optimization and security -- NGINX configuration and routing -- Observability stacks (Grafana, Prometheus, Loki) -- SBOM and CVE mitigation workflows -- Geospatial data engineering -- ML/AI deployment patterns - -**These documents answer "how"**—they describe proven patterns and implementation details. - -**What's missing is "why"**—the context, trade-offs, and rationale behind each decision. - -### Problems Without ADRs - -1. **Decisions live in ephemeral channels**: Chat, email, or someone's memory -2. **Repeated questions**: New contributors ask "Why Postgres here and not X?", "Why Loki over ELK?", "Why PgAudit instead of native logging?" -3. **Architectural thrash**: Teams make conflicting changes because original decisions are forgotten -4. **Onboarding friction**: New engineers spend weeks understanding "why things are this way" -5. **Compliance gaps**: Auditors ask "Why this security posture?" and there's no documented answer - -### The Value of ADRs - -ADRs provide: -- **Historical context**: Why a decision was made at a specific point in time -- **Trade-off transparency**: What alternatives were considered and why they were rejected -- **Decision audit trail**: Who decided, when, and what the expected consequences were -- **Living documentation**: Decisions can be superseded, but history is preserved - -### Example: The PgAudit Decision - -**Without ADR**: A new engineer sees PgAudit configured and wonders: -- "Why not use native PostgreSQL logging?" -- "Why CSV logs instead of JSON?" -- "Why PgCron for rotation instead of logrotate?" - -**With ADR**: ADR-0005 documents: -- Context: Compliance requirements for audit trails -- Options: Native logging, PgAudit, third-party tools -- Decision: PgAudit + CSV logs + PgCron -- Rationale: Structured parsing, automated rotation, database-native scheduling -- Consequences: Performance overhead, maintenance complexity, compliance coverage - -## ADR Basics: What, When, and How Much Detail - -### What Is an ADR? - -An **Architecture Decision Record (ADR)** is a short, versioned document that captures: -- **A single significant decision** -- **The context** that led to the decision -- **Options considered** and why they were rejected -- **The decision** itself -- **Expected consequences** (positive and negative) - -### When to Write an ADR - -Write an ADR when you make a decision that: -- **Introduces a new major technology** (e.g., switching from ELK to Loki) -- **Changes security posture** (e.g., adopting SBOM/Trivy policy) -- **Affects data model or storage architecture** (e.g., choosing Parquet over JSONB) -- **Defines a standard pattern** (e.g., GitFlow workflow, NGINX routing strategy) -- **Will be questioned later** ("Why did we choose X instead of Y?") - -### When NOT to Write an ADR - -Skip ADRs for: -- **Trivial refactors**: Code cleanup, variable renaming -- **Local implementation details**: Covered by code comments -- **Temporary workarounds**: Documented in issue trackers -- **Obvious choices**: When there's only one reasonable option - -### Decision Checklist: "Should This Be an ADR?" - -Ask yourself: -1. **Will a future engineer ask "why?"** about this decision? -2. **Does this affect multiple systems or teams?** -3. **Are there multiple reasonable alternatives?** -4. **Will this decision be referenced in design docs or runbooks?** -5. **Does this change our security, compliance, or operational posture?** -6. **Is this a pattern we'll reuse across projects?** -7. **Would reversing this decision be expensive or disruptive?** - -If you answer "yes" to 3+ questions, write an ADR. - -## ADR Repository Structure & Naming - -### Recommended Structure - -**For Documentation Repositories** (like `sempervent.github.io`): - -``` -/docs - /adr - /_templates - 0001-template-standard.md - 0002-template-lightweight.md - 0001-use-postgres-for-audit-log.md - 0002-standardize-on-openmaptiles-for-basemaps.md - 0003-sbom-trivy-policy-for-docker-images.md - 0004-observability-stack-standardization.md - 0005-pgaudit-pgcron-audit-pipeline.md - index.md -``` - -**For Code Repositories**: - -``` -/adr - 0001-title.md - 0002-title.md - index.md -``` - -### Naming Convention - -**Format**: `NNNN-kebab-case-title.md` - -- **Sequential numeric prefix**: `0001-`, `0002-`, etc. (padded to 4 digits) -- **Kebab-case title**: Lowercase, hyphens for spaces -- **No gaps**: If ADR-0005 is deleted, don't reuse the number; keep sequence continuous - -**Examples**: -- `0001-use-postgres-for-audit-log.md` -- `0002-standardize-on-openmaptiles-for-basemaps.md` -- `0003-sbom-trivy-policy-for-docker-images.md` - -### Status Labels - -Each ADR must have a status: - -- **`Proposed`**: Draft, under review, not yet accepted -- **`Accepted`**: Decision is active and should be followed -- **`Superseded`**: Replaced by a newer ADR (include link to superseding ADR) -- **`Rejected`**: Decision was not adopted (include reason) -- **`Deprecated`**: Decision is no longer relevant but kept for historical context - -### Linking Related ADRs - -Use explicit links in ADR documents: - -```markdown -## Related ADRs - -- **Supersedes**: [ADR-0003: Legacy Observability Stack](../adr/0003-legacy-observability-stack.md) -- **Superseded by**: [ADR-0010: Unified Observability Platform](../adr/0010-unified-observability-platform.md) -- **Related**: [ADR-0005: Database Auditing Strategy](../adr/0005-database-auditing-strategy.md) -``` - -## ADR Templates - -### Standard ADR Template - -Use this template for most architectural decisions: - -```markdown -# ADR-XXXX: [Short Descriptive Title] - -**Status**: Proposed | Accepted | Superseded | Rejected | Deprecated - -**Date**: YYYY-MM-DD - -**Deciders**: [Names or roles, e.g., "Architecture Working Group", "Tech Leads"] - -**Tags**: [Optional: component, domain, e.g., "observability", "security", "data"] - ---- - -## Context - -[2-4 paragraphs describing the situation, problem, or requirement that led to this decision. Include: -- What problem are we solving? -- What constraints exist? -- What are the business/technical drivers?] - -## Decision - -[1-2 paragraphs stating the decision clearly and concisely. Be specific about what is being decided.] - -We will [specific action/choice]. - -## Options Considered - -### Option 1: [Name of Option] - -**Description**: [What this option entails] - -**Pros**: -- [Benefit 1] -- [Benefit 2] - -**Cons**: -- [Drawback 1] -- [Drawback 2] - -### Option 2: [Name of Option] - -[Same structure as Option 1] - -### Option 3: [Name of Option] - -[Same structure as Option 1] - -## Rationale - -[2-3 paragraphs explaining why the chosen option was selected. Reference specific pros/cons, constraints, or requirements that drove the decision.] - -## Consequences - -### Positive - -- [Expected positive outcome 1] -- [Expected positive outcome 2] - -### Negative - -- [Expected negative outcome 1] -- [Expected negative outcome 2] - -### Neutral / Trade-offs - -- [Trade-off or neutral consequence] - -## Related Documents - -- [Link to design doc, RFC, or implementation guide] -- [Link to best-practices document] -- [Link to runbook or operational guide] - -## Implementation Notes - -[Optional: Brief notes on how this decision will be implemented, key milestones, or dependencies] - -## References - -[Optional: Links to external resources, research, or benchmarks that informed the decision] -``` - -### Lightweight ADR Template - -Use this template for smaller, focused decisions: - -```markdown -# ADR-XXXX: [Short Descriptive Title] - -**Status**: Proposed | Accepted | Superseded | Rejected | Deprecated - -**Date**: YYYY-MM-DD - -**Deciders**: [Names or roles] - ---- - -## Context - -[1-2 paragraphs: What problem or situation led to this decision?] - -## Decision - -[1 paragraph: What is being decided?] - -We will [specific action/choice]. - -## Impact / Notes - -[1-2 paragraphs: What are the key consequences, trade-offs, or implementation considerations?] - -## Related Documents - -- [Link to relevant docs] -``` - -## Decision Scope & Granularity - -### Best Practices - -**Each ADR should cover one main decision**. Avoid bundling unrelated choices into a single ADR. - -### Scope Guidelines - -| Scope | Example | Good/Bad | -|-------|---------|----------| -| **Too Big** | "Observability, Logging, Monitoring, and All Data Pipelines v3" | ❌ Too broad, multiple unrelated decisions | -| **Just Right** | "Standardize on Grafana + Prometheus + Loki for Observability" | ✅ Single coherent decision | -| **Just Right** | "Use PgAudit + CSV Logs + PgCron for Audit Logging Pipeline" | ✅ Related components, single purpose | -| **Just Right** | "Adopt GitFlow with Protected Main/Develop and Release Branches" | ✅ Single workflow decision | -| **Just Right** | "Generate Dark OpenMapTiles for US at Z≤12 as Default Basemap" | ✅ Single technology/pattern decision | -| **Too Small** | "Use `asyncpg` instead of `psycopg2` in service X" | ❌ Implementation detail, not architectural | - -### Good ADR Examples (From This Ecosystem) - -1. **"Standardize on Grafana + Prometheus + Loki for Observability"** - - Single decision: Which observability stack? - - Clear scope: Metrics, logs, dashboards - - Related but distinct: Could have separate ADRs for alerting, tracing - -2. **"Use PgAudit + CSV Logs + PgCron for Audit Logging Pipeline"** - - Single decision: How to implement database auditing? - - Related components: All part of one pipeline - - Clear boundary: Database auditing, not application logging - -3. **"Adopt GitFlow with Protected Main/Develop and Release Branches"** - - Single decision: Which branching model? - - Clear scope: Version control workflow - - Related but distinct: Could have separate ADR for CI/CD integration - -4. **"Generate Dark OpenMapTiles for US at Z≤12 as Default Basemap"** - - Single decision: Which basemap technology and configuration? - - Clear scope: Geospatial visualization - - Related but distinct: Could have separate ADR for tile serving infrastructure - -### When to Split or Combine - -**Split if**: -- ADR covers multiple unrelated systems (e.g., "Database and Message Queue Strategy") -- Different teams own different parts -- Decisions can be made independently - -**Combine if**: -- Components are tightly coupled (e.g., PgAudit + CSV logs + PgCron) -- One decision depends on another -- They form a single coherent pattern - -## Integrating ADRs with Git & Pull Requests - -### Workflow: Creating a New ADR - -**Step 1: Create Branch** - -```bash -# Create feature branch -git checkout -b adr/0007-sbom-trivy-policy - -# Or use descriptive name -git checkout -b adr/0007-docker-image-security-policy -``` - -**Step 2: Create ADR File** - -```bash -# Create ADR file -touch docs/adr/0007-sbom-trivy-policy-for-docker-images.md - -# Copy template -cp docs/adr/_templates/0001-template-standard.md docs/adr/0007-sbom-trivy-policy-for-docker-images.md -``` - -**Step 3: Draft ADR** - -- Set status to `Proposed` -- Fill in context, options, decision, rationale -- Link to related documents - -**Step 4: Update ADR Index** - -```markdown -# Architecture Decision Records - -| Number | Title | Status | Date | -|--------|-------|--------|------| -| 0007 | SBOM and Trivy Policy for Docker Images | Proposed | 2024-01-15 | -``` - -### Workflow: Review Process - -**Step 1: Open Pull Request** - -**PR Title**: `ADR-0007: SBOM and Trivy Policy for Docker Images` - -**PR Description Template**: - -```markdown -## ADR Proposal - -This PR proposes [brief summary of decision]. - -**Status**: Proposed - -**Deciders**: @tech-lead-1, @security-lead, @ops-lead - -**Related Issues**: #123, #456 - -## Review Checklist - -- [ ] Context and problem statement are clear -- [ ] All reasonable options have been considered -- [ ] Rationale is well-documented -- [ ] Consequences (positive and negative) are identified -- [ ] Related documents are linked -- [ ] ADR index is updated - -## Questions for Reviewers - -- [Question 1] -- [Question 2] -``` - -**Step 2: Request Review** - -Request review from: -- **Tech leads / architects**: Validate technical soundness -- **Stakeholders**: Security, ops, data teams affected by decision -- **Domain experts**: People with relevant expertise - -**Step 3: Address Feedback** - -- Update ADR based on review comments -- Document any changes in PR comments -- Keep status as `Proposed` until consensus - -**Step 4: Acceptance** - -Once approved: -- Change status to `Accepted` -- Merge PR -- Optionally tag repository with release that includes decision - -### Commit Message Convention - -**Format**: `docs(adr): add ADR-XXXX: [Title]` - -**Examples**: - -```bash -git commit -m "docs(adr): add ADR-0007: SBOM and Trivy Policy for Docker Images - -- Proposes mandatory SBOM generation for all Docker images -- Requires Trivy scan with CRITICAL/HIGH severity gates -- Establishes automated CVE mitigation workflow - -Status: Proposed" -``` - -### Workflow: Superseding an ADR - -**Step 1: Create New ADR** - -```bash -git checkout -b adr/0010-unified-observability-platform -touch docs/adr/0010-unified-observability-platform.md -``` - -**Step 2: Reference Old ADR** - -In new ADR: - -```markdown -## Related ADRs - -- **Supersedes**: [ADR-0004: Observability Stack Standardization](../adr/0004-observability-stack-standardization.md) -``` - -**Step 3: Update Old ADR** - -In old ADR: - -```markdown -**Status**: Superseded - -## Related ADRs - -- **Superseded by**: [ADR-0010: Unified Observability Platform](../adr/0010-unified-observability-platform.md) -``` - -**Step 4: Merge Both Changes** - -- New ADR: Status `Accepted` -- Old ADR: Status `Superseded` - -## Linking ADRs to Best-Practices & Runbooks - -### Best-Practices Documents Should Reference ADRs - -**Pattern**: Add ADR references at the top or bottom of best-practices documents. - -**Example: NGINX Best Practices** - -```markdown -# NGINX Best Practices: Patterns, Hardening, and Multi-API Replication - -**This document is guided by**: [ADR-0003: NGINX as Edge Reverse Proxy for Internal APIs](../adr/0003-nginx-edge-reverse-proxy.md) - -[Rest of document...] - ---- - -## Related Documents - -- [ADR-0003: NGINX as Edge Reverse Proxy](../adr/0003-nginx-edge-reverse-proxy.md) -- [ADR-0008: Multi-API Gateway Pattern](../adr/0008-multi-api-gateway-pattern.md) -``` - -**Example: Observability Best Practices** - -```markdown -# Best Practices for Monitoring & Observability - -**This approach is defined by**: [ADR-0004: Observability Stack Standardization](../adr/0004-observability-stack-standardization.md) - -[Rest of document...] -``` - -### ADRs Should Link to Implementation Guides - -In ADR's "Related Documents" section: - -```markdown -## Related Documents - -- **[Best Practices: Grafana, Prometheus, Loki](../operations-monitoring/grafana-prometheus-loki-observability.md)** - Implementation guide -- **[Tutorial: Setting Up Prometheus Scraping](../tutorials/operations-monitoring/prometheus-setup.md)** - Step-by-step setup -- **[Runbook: Observability Stack Incident Response](../runbooks/observability-incident-response.md)** - Operational procedures -``` - -### Handling Divergence - -If best-practices diverge from ADR: - -```markdown -## Note on ADR Alignment - -This document follows [ADR-0004: Observability Stack Standardization](../adr/0004-observability-stack-standardization.md) with one exception: - -**Divergence**: We use Promtail instead of Fluentd for log ingestion. - -**Reason**: Promtail provides better integration with Loki and reduces operational complexity in our Kubernetes environment. - -**Impact**: This is an implementation detail that doesn't change the core decision in ADR-0004. -``` - -## Decision Lifecycle & Governance - -### Governance Model - -**Who Can Propose ADRs?** - -- **Any engineer** can propose an ADR -- No gatekeeping on proposal; all proposals are welcome - -**Who Can Approve ADRs?** - -- **Architecture Working Group** (or equivalent): For system-wide decisions -- **Tech Leads Council**: For cross-team decisions -- **Domain Experts**: For domain-specific decisions (e.g., security team for security ADRs) -- **Consensus-based**: For smaller teams, require approval from 2-3 relevant stakeholders - -**Decision Authority Matrix**: - -| Decision Scope | Approver | Example | -|----------------|----------|---------| -| **System-wide** | Architecture Working Group | Observability stack, database strategy | -| **Security** | Security Team + Tech Leads | SBOM policy, audit logging | -| **Domain-specific** | Domain Experts + Tech Leads | Geospatial basemap, ML deployment | -| **Team-specific** | Team Tech Lead | Service-specific patterns | - -### Decision Lifecycle - -```mermaid -graph LR - A[Proposed] --> B{In Review} - B -->|Approved| C[Accepted] - B -->|Rejected| D[Rejected] - C -->|Superseded| E[Superseded] - C -->|No longer relevant| F[Deprecated] - - style A fill:#fff4cc - style B fill:#cfe2f3 - style C fill:#d9ead3 - style D fill:#f4cccc - style E fill:#e1d5e7 - style F fill:#fce5cd -``` - -**States**: -- **Proposed**: Draft, under review -- **In Review**: Actively being discussed (optional explicit state) -- **Accepted**: Decision is active -- **Rejected**: Decision was not adopted -- **Superseded**: Replaced by newer ADR -- **Deprecated**: No longer relevant but kept for history - -### Review Cadence - -**Quarterly ADR Review Meeting**: - -**Agenda**: -1. **Review stale ADRs**: Identify ADRs that may need updating -2. **Mark as Deprecated**: ADRs for systems that no longer exist -3. **Identify Superseded ADRs**: Find ADRs that have been implicitly replaced -4. **Update statuses**: Ensure all ADRs have correct status -5. **Archive old ADRs**: Move deprecated ADRs to archive (optional) - -**Process**: -- Tech leads review ADR index -- Identify candidates for status changes -- Discuss in meeting -- Update ADRs via PRs - -### Handling Disagreements - -**Use ADRs to make disagreements explicit**: - -1. **Capture competing options**: Document all serious alternatives -2. **Record rationale**: Explain why each option was considered -3. **Document trade-offs**: Be explicit about what's being traded off -4. **Record the compromise**: The decision is the documented compromise - -**Example**: - -```markdown -## Options Considered - -### Option 1: ELK Stack (Elasticsearch, Logstash, Kibana) - -**Pros**: Mature, feature-rich, excellent search capabilities - -**Cons**: Resource-intensive, complex to operate, licensing costs at scale - -### Option 2: Grafana + Prometheus + Loki - -**Pros**: Lightweight, cost-effective, excellent Grafana integration - -**Cons**: Less mature than ELK, different query language (LogQL vs Lucene) - -## Rationale - -After evaluation, we chose Option 2 (Grafana + Prometheus + Loki) because: -- Cost constraints favor open-source, self-hosted solutions -- Existing Prometheus investment reduces operational overhead -- LogQL, while different, is sufficient for our use cases -- Team familiarity with Grafana reduces training burden - -**Trade-off**: We sacrifice some advanced search capabilities for operational simplicity and cost efficiency. -``` - -## Concrete Examples - -### Example 1: Standardize on PgAudit + CSV Logs + PgCron for Database Auditing - -```markdown -# ADR-0005: Use PgAudit + CSV Logs + PgCron for Database Auditing - -**Status**: Accepted - -**Date**: 2024-01-15 - -**Deciders**: Database Team, Security Team, Architecture Working Group - -**Tags**: database, security, compliance - ---- - -## Context - -We need comprehensive audit logging for PostgreSQL databases to meet compliance requirements (SOC2, GDPR). The audit trail must: -- Log all DDL, DML, and function calls -- Be queryable and analyzable -- Rotate automatically to manage disk space -- Integrate with our existing observability stack - -Current PostgreSQL native logging provides basic query logging but lacks structured audit events and automated rotation capabilities. - -## Decision - -We will use **PgAudit** for structured audit logging, **CSV logs** for parsing and ingestion, and **PgCron** for automated log rotation and data refresh. - -## Options Considered - -### Option 1: Native PostgreSQL Logging - -**Description**: Use PostgreSQL's built-in `log_statement` and `log_min_duration_statement` settings. - -**Pros**: -- No additional extensions required -- Simple configuration -- Low overhead - -**Cons**: -- No structured audit events (just query text) -- Difficult to parse and analyze -- No automated rotation (requires external tools) -- Limited filtering capabilities - -### Option 2: PgAudit + External Log Rotation - -**Description**: Use PgAudit for audit logging, but rely on OS-level logrotate for rotation. - -**Pros**: -- Structured audit events -- Standard OS tooling -- No database extension for rotation - -**Cons**: -- Requires coordination between database and OS -- Less flexible for database-specific retention policies -- Harder to integrate with database-driven refresh jobs - -### Option 3: PgAudit + CSV Logs + PgCron (Chosen) - -**Description**: Use PgAudit for audit events, CSV logs for structured parsing, and PgCron for automated rotation and data refresh. - -**Pros**: -- Structured audit events via PgAudit -- CSV format enables easy parsing and ingestion -- Database-native scheduling via PgCron -- Can build analytical views on audit data -- Automated refresh keeps audit tables current - -**Cons**: -- Requires two extensions (pgaudit, pg_cron) -- Performance overhead from audit logging (5-15%) -- More complex setup and maintenance - -## Rationale - -We chose Option 3 because: -- **Compliance requirements** demand structured, queryable audit trails -- **CSV logs** enable us to build analytical views (top users, top tables, activity logs) -- **PgCron** provides database-native scheduling, reducing operational complexity -- **Integrated approach** allows us to refresh audit tables automatically and build dashboards - -The performance overhead is acceptable given compliance requirements, and we can tune PgAudit settings (e.g., `pgaudit.log = 'ddl,write'` instead of `'all'`) to reduce impact. - -## Consequences - -### Positive - -- Comprehensive audit trail for compliance -- Queryable audit data via database views -- Automated rotation and refresh reduce operational burden -- Can build dashboards and alerts on audit data - -### Negative - -- Performance overhead (5-15% depending on logging level) -- Additional maintenance for PgAudit and PgCron extensions -- Storage requirements for audit logs and parsed data - -### Neutral / Trade-offs - -- More complex setup, but better long-term maintainability -- Database-native scheduling vs OS-level tooling - -## Related Documents - -- **[Best Practices: PostgreSQL Security](../postgres/postgres-security-best-practices.md)** -- **[Tutorial: Auditing PostgreSQL with PgAudit and PgCron](../../tutorials/database-data-engineering/postgres-pgaudit-pgcron-auditing.md)** - -## Implementation Notes - -- Enable PgAudit and PgCron in `postgresql.conf` -- Configure CSV logging with appropriate rotation settings -- Create audit schema with staging and parsed log tables -- Build refresh function and analytical views -- Schedule PgCron jobs for rotation and refresh -``` - -### Example 2: Adopt Grafana + Prometheus + Loki as Default Observability Stack - -```markdown -# ADR-0004: Standardize on Grafana + Prometheus + Loki for Observability - -**Status**: Accepted - -**Date**: 2024-01-10 - -**Deciders**: Architecture Working Group, Operations Team - -**Tags**: observability, monitoring, logging - ---- - -## Context - -We need a unified observability stack for metrics, logs, and dashboards across all systems (infrastructure, applications, data pipelines). Current state: -- Metrics scattered across multiple systems -- Logs in various formats and locations -- No unified dashboarding solution -- High operational overhead from multiple tools - -Requirements: -- Cost-effective (prefer open-source, self-hosted) -- Scalable to hundreds of services -- Easy integration with existing systems -- Strong visualization capabilities - -## Decision - -We will standardize on **Grafana** for dashboards, **Prometheus** for metrics, and **Loki** for logs. All new systems must integrate with this stack. - -## Options Considered - -### Option 1: ELK Stack (Elasticsearch, Logstash, Kibana) - -**Description**: Use Elasticsearch for logs, Logstash for processing, Kibana for dashboards. Add Prometheus for metrics. - -**Pros**: -- Mature, feature-rich -- Excellent search capabilities (Lucene) -- Strong ecosystem - -**Cons**: -- Resource-intensive (Elasticsearch requires significant RAM/CPU) -- Complex to operate at scale -- Licensing costs for advanced features -- Different stack for logs vs metrics - -### Option 2: Datadog / New Relic / Splunk (SaaS) - -**Description**: Use commercial SaaS observability platform. - -**Pros**: -- Fully managed, no operational overhead -- Excellent UI and features -- Strong support - -**Cons**: -- High cost at scale (per-host or per-GB pricing) -- Vendor lock-in -- Data residency concerns for sensitive data -- Limited customization - -### Option 3: Grafana + Prometheus + Loki (Chosen) - -**Description**: Use Grafana for visualization, Prometheus for metrics, Loki for logs. - -**Pros**: -- Cost-effective (open-source, self-hosted) -- Unified stack (metrics and logs in Grafana) -- PromQL and LogQL are powerful query languages -- Excellent Grafana integration -- Can reuse existing Prometheus investment - -**Cons**: -- Less mature than ELK for log search -- LogQL different from Lucene (learning curve) -- Requires operational expertise - -## Rationale - -We chose Option 3 because: -- **Cost constraints** favor open-source, self-hosted solutions -- **Unified stack** reduces operational complexity (one dashboard tool) -- **Existing Prometheus investment** means we can leverage current setup -- **LogQL**, while different from Lucene, is sufficient for our use cases -- **Team familiarity** with Grafana reduces training burden - -The trade-off of less advanced log search is acceptable given cost and operational simplicity benefits. - -## Consequences - -### Positive - -- Unified observability stack reduces operational overhead -- Cost-effective (no per-host or per-GB SaaS fees) -- Strong Grafana integration for metrics and logs -- Can scale horizontally - -### Negative - -- Requires operational expertise to run Prometheus and Loki -- LogQL learning curve for teams used to Lucene -- Less mature log search than ELK - -### Neutral / Trade-offs - -- Self-hosted vs managed: More control, more operational burden -- LogQL vs Lucene: Different but sufficient for our needs - -## Related Documents - -- **[Best Practices: Monitoring & Observability](../operations-monitoring/grafana-prometheus-loki-observability.md)** -- **[Tutorial: Setting Up Prometheus Scraping](../../tutorials/operations-monitoring/prometheus-setup.md)** - -## Implementation Notes - -- Deploy Prometheus for metrics collection -- Deploy Loki for log aggregation -- Deploy Grafana with Prometheus and Loki datasources -- Migrate existing dashboards to Grafana -- Train teams on PromQL and LogQL -``` - -### Example 3: Use OpenMapTiles Dark Basemap (US z≤12) for All Geospatial Dashboards - -```markdown -# ADR-0002: Standardize on OpenMapTiles Dark Basemap for Geospatial Dashboards - -**Status**: Accepted - -**Date**: 2024-01-05 - -**Deciders**: Data Engineering Team, Frontend Team - -**Tags**: geospatial, visualization, frontend - ---- - -## Context - -We need a consistent basemap for all geospatial dashboards (Grafana, custom web apps, internal tools). Requirements: -- Dark theme (reduces eye strain, matches our dashboard aesthetic) -- Covers entire United States -- Vector tiles (for performance and customization) -- Self-hosted (for air-gapped environments and cost control) -- Zoom level 12 sufficient for strategic/overview maps - -Current state: Mixed basemaps (some use Mapbox, some use OpenStreetMap, some have no basemap). - -## Decision - -We will generate and self-host **dark-themed OpenMapTiles** covering the entire United States at zoom level 12, and use this as the default basemap for all geospatial dashboards. - -## Options Considered - -### Option 1: Mapbox (SaaS) - -**Description**: Use Mapbox's hosted vector tiles with dark style. - -**Pros**: -- Fully managed, no operational overhead -- High-quality tiles and styles -- Easy integration - -**Cons**: -- Cost at scale (per-request pricing) -- Not suitable for air-gapped environments -- Vendor lock-in - -### Option 2: OpenStreetMap Raster Tiles - -**Description**: Use standard OSM raster tiles (e.g., from tile.openstreetmap.org). - -**Pros**: -- Free, no operational overhead -- Simple integration - -**Cons**: -- Not dark-themed -- Raster tiles (larger, less customizable) -- Rate limiting on public servers -- Not self-hosted - -### Option 3: OpenMapTiles Dark (Self-Hosted) (Chosen) - -**Description**: Generate dark-themed OpenMapTiles for US at z≤12, host via tileserver-gl. - -**Pros**: -- Self-hosted (air-gapped compatible, no per-request costs) -- Dark theme matches dashboard aesthetic -- Vector tiles (smaller, more customizable) -- Full control over style and data - -**Cons**: -- Requires initial tile generation (time and storage) -- Operational overhead (hosting tileserver) -- Limited to z≤12 (may need higher zoom for specific regions) - -## Rationale - -We chose Option 3 because: -- **Air-gapped environments** require self-hosted solutions -- **Dark theme** matches our dashboard aesthetic and reduces eye strain -- **Cost control**: No per-request fees, one-time generation cost -- **Zoom level 12** is sufficient for strategic/overview maps (can generate higher zoom for specific regions if needed) - -The operational overhead is acceptable given cost and flexibility benefits. - -## Consequences - -### Positive - -- Consistent basemap across all dashboards -- Dark theme reduces eye strain -- Self-hosted enables air-gapped deployments -- No per-request costs - -### Negative - -- Initial tile generation requires time and storage -- Operational overhead for tileserver -- Limited to z≤12 (may need region-specific higher zoom tiles) - -### Neutral / Trade-offs - -- Self-hosted vs SaaS: More control, more operational burden -- z≤12 vs higher zoom: Balances coverage with storage/performance - -## Related Documents - -- **[Tutorial: Generating Dark OpenMapTiles for US at Z12](../../tutorials/database-data-engineering/openmaptiles-us-dark-z12.md)** -- **[Best Practices: Geospatial Data Engineering](../database-data/geospatial-data-engineering.md)** - -## Implementation Notes - -- Generate US-wide tiles at z≤12 using OpenMapTiles toolchain -- Deploy tileserver-gl for tile serving -- Create dark style JSON -- Integrate into Grafana and custom web apps -- Document tile generation process for future updates -``` - -### Example 4: Require SBOM + Trivy Scan for All Docker Images Before Release - -```markdown -# ADR-0003: SBOM and Trivy Policy for Docker Images - -**Status**: Accepted - -**Date**: 2024-01-12 - -**Deciders**: Security Team, DevOps Team, Architecture Working Group - -**Tags**: security, containers, compliance - ---- - -## Context - -We need to ensure container security and compliance with supply-chain security requirements. Current state: -- Docker images built and deployed without security scanning -- No Software Bill of Materials (SBOM) for containers -- No automated CVE detection or mitigation -- Compliance requirements (SOC2, SLSA) demand supply-chain visibility - -Requirements: -- Generate SBOMs for all Docker images -- Scan images for CVEs before release -- Automate CVE mitigation where possible -- Fail builds on critical vulnerabilities - -## Decision - -We will require **SBOM generation** and **Trivy scanning** for all Docker images before release. Builds will fail on CRITICAL or HIGH severity CVEs unless explicitly allowed via ignore list. - -## Options Considered - -### Option 1: Manual Scanning (Current State) - -**Description**: Developers manually run security scans before releases. - -**Pros**: -- No tooling changes required -- Flexible process - -**Cons**: -- Inconsistent application -- Easy to forget or skip -- No automation -- No SBOM generation - -### Option 2: GitHub/GitLab Native Scanners - -**Description**: Use built-in security scanning (e.g., GitHub Dependabot, GitLab Container Scanning). - -**Pros**: -- Integrated into existing workflows -- No additional tooling - -**Cons**: -- Limited to specific platforms -- Less control over policies -- May not generate SBOMs in required format -- Less flexible for custom workflows - -### Option 3: Trivy + Syft (Chosen) - -**Description**: Use Trivy for vulnerability scanning and Syft for SBOM generation, integrated into CI/CD pipelines. - -**Pros**: -- Comprehensive scanning (vulnerabilities, misconfigurations, secrets) -- SBOM generation in multiple formats (CycloneDX, SPDX) -- Flexible policy enforcement -- Works with any CI/CD system -- Can automate CVE mitigation - -**Cons**: -- Requires CI/CD integration -- Operational overhead for policy management -- May require base image updates to fix CVEs - -## Rationale - -We chose Option 3 because: -- **Comprehensive scanning**: Trivy covers vulnerabilities, misconfigurations, and secrets -- **SBOM generation**: Syft produces SBOMs in standard formats (CycloneDX, SPDX) for compliance -- **Flexible policies**: Can enforce different policies per environment (warn in dev, fail in prod) -- **Automation potential**: Can integrate with dependency update tools (Renovate, Dependabot) for auto-fix -- **Platform-agnostic**: Works with any CI/CD system, not tied to specific platforms - -The operational overhead is acceptable given security and compliance benefits. - -## Consequences - -### Positive - -- Comprehensive security scanning before release -- SBOMs enable supply-chain visibility and compliance -- Automated policy enforcement reduces human error -- Can integrate with automated CVE mitigation - -### Negative - -- CI/CD pipeline changes required -- May block releases until CVEs are fixed -- Operational overhead for policy management -- Requires base image and dependency updates - -### Neutral / Trade-offs - -- Security vs velocity: May slow releases, but improves security posture -- Automation vs flexibility: More automated, but requires policy definition - -## Related Documents - -- **[Best Practices: SBOMs, Trivy Scans, and Automated CVE Mitigation](../docker-infrastructure/docker-sbom-trivy-cve-mitigation.md)** -- **[Tutorial: Setting Up Trivy in CI/CD](../../tutorials/docker-infrastructure/trivy-ci-cd-setup.md)** - -## Implementation Notes - -- Integrate Trivy and Syft into CI/CD pipelines -- Configure severity thresholds (fail on CRITICAL/HIGH) -- Set up SBOM artifact storage -- Create ignore list process for unavoidable CVEs -- Document policy exceptions and approval process -``` - -## Tooling & Automation - -### Pre-commit / CI Checks - -**Validate ADR Structure**: - -Create a simple script to validate ADR files: - -```bash -#!/bin/bash -# scripts/validate-adr.sh - -adr_file="$1" - -# Check required fields -required_fields=("Status" "Date" "Deciders") -missing_fields=() - -for field in "${required_fields[@]}"; do - if ! grep -q "^\*\*${field}\*\*:" "$adr_file"; then - missing_fields+=("$field") - fi -done - -if [ ${#missing_fields[@]} -ne 0 ]; then - echo "ERROR: Missing required fields in $adr_file: ${missing_fields[*]}" - exit 1 -fi - -# Check status is valid -status=$(grep "^\*\*Status\*\*:" "$adr_file" | sed 's/.*: //') -valid_statuses=("Proposed" "Accepted" "Superseded" "Rejected" "Deprecated") -if [[ ! " ${valid_statuses[@]} " =~ " ${status} " ]]; then - echo "ERROR: Invalid status '$status' in $adr_file" - exit 1 -fi - -# Check naming convention -filename=$(basename "$adr_file") -if ! [[ "$filename" =~ ^[0-9]{4}-.*\.md$ ]]; then - echo "ERROR: ADR filename must match pattern NNNN-kebab-case-title.md" - exit 1 -fi - -echo "✓ ADR $adr_file is valid" -``` - -**GitHub Actions Example**: - -```yaml -# .github/workflows/validate-adrs.yml -name: Validate ADRs - -on: - pull_request: - paths: - - 'docs/adr/*.md' - -jobs: - validate: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Validate ADRs - run: | - for adr in docs/adr/*.md; do - if [[ "$adr" != "docs/adr/index.md" ]] && [[ "$adr" != "docs/adr/_templates"* ]]; then - ./scripts/validate-adr.sh "$adr" - fi - done -``` - -### ADR Index Generation - -**Auto-Generate Index**: - -```bash -#!/bin/bash -# scripts/generate-adr-index.sh - -index_file="docs/adr/index.md" -adr_dir="docs/adr" - -cat > "$index_file" << 'EOF' -# Architecture Decision Records - -This directory contains Architecture Decision Records (ADRs) for this project. - -See [ADR Best Practices](../../best-practices/architecture-design/adr-decision-governance.md) for guidelines on creating and maintaining ADRs. - -## ADR Index - -| Number | Title | Status | Date | Deciders | -|--------|-------|--------|------|----------| -EOF - -# Extract ADR metadata -for adr in "$adr_dir"/*.md; do - if [[ "$adr" == "$index_file" ]] || [[ "$adr" == *"/_templates/"* ]]; then - continue - fi - - number=$(basename "$adr" | cut -d'-' -f1) - title=$(grep "^# ADR-" "$adr" | sed 's/^# ADR-[0-9]*: //') - status=$(grep "^\*\*Status\*\*:" "$adr" | sed 's/.*: //') - date=$(grep "^\*\*Date\*\*:" "$adr" | sed 's/.*: //') - deciders=$(grep "^\*\*Deciders\*\*:" "$adr" | sed 's/.*: //' | sed 's/|/\\|/g') - - if [ -n "$title" ]; then - echo "| $number | [$title]($(basename "$adr")) | $status | $date | $deciders |" >> "$index_file" - fi -done - -echo "✓ Generated ADR index at $index_file" -``` - -**Make Target**: - -```makefile -# Makefile - -.PHONY: adr-index -adr-index: - @./scripts/generate-adr-index.sh - -.PHONY: validate-adrs -validate-adrs: - @for adr in docs/adr/*.md; do \ - if [[ "$$adr" != "docs/adr/index.md" ]] && [[ "$$adr" != *"/_templates/"* ]]; then \ - ./scripts/validate-adr.sh "$$adr"; \ - fi \ - done -``` - -### Tagging and Labeling - -**GitHub/GitLab Labels**: - -Create labels for ADR-related issues and PRs: -- `adr` - Architecture Decision Record -- `architecture` - Architectural change -- `adr/proposed` - ADR under review -- `adr/accepted` - ADR accepted - -**PR Template**: - -```markdown -## Type of Change - -- [ ] Bug fix -- [ ] New feature -- [ ] ADR (Architecture Decision Record) -- [ ] Documentation -- [ ] Other - -## ADR Details (if applicable) - -- ADR Number: ADR-XXXX -- Status: Proposed | Accepted -- Related Issues: #123 -``` - -### Dashboards and Search - -**ADR Metadata Extraction**: - -For advanced use cases, extract ADR metadata into a searchable format: - -```python -# scripts/extract-adr-metadata.py -import re -import json -from pathlib import Path - -adr_dir = Path("docs/adr") -adrs = [] - -for adr_file in adr_dir.glob("*.md"): - if adr_file.name == "index.md": - continue - - content = adr_file.read_text() - - adr = { - "number": re.search(r"ADR-(\d+)", content).group(1) if re.search(r"ADR-(\d+)", content) else None, - "title": re.search(r"# ADR-\d+: (.+)", content).group(1) if re.search(r"# ADR-\d+: (.+)", content) else None, - "status": re.search(r"\*\*Status\*\*: (.+)", content).group(1).strip() if re.search(r"\*\*Status\*\*: (.+)", content) else None, - "date": re.search(r"\*\*Date\*\*: (.+)", content).group(1).strip() if re.search(r"\*\*Date\*\*: (.+)", content) else None, - "tags": re.findall(r"\*\*Tags\*\*: (.+)", content)[0].split(", ") if re.search(r"\*\*Tags\*\*: (.+)", content) else [], - "file": str(adr_file.relative_to(Path("."))) - } - - adrs.append(adr) - -# Output as JSON for search/indexing -print(json.dumps(adrs, indent=2)) -``` - -## Common Pitfalls & Anti-Patterns - -### Pitfall 1: Writing ADRs Long After Decision - -**Symptom**: ADR is written months or years after the decision was made. - -**Why It's a Problem**: -- Context is lost or misremembered -- Alternatives considered are forgotten -- Rationale becomes post-hoc justification - -**How to Prevent**: -- Write ADR **during or immediately after** decision-making -- Make ADR part of the decision process, not an afterthought -- If retroactively documenting, acknowledge uncertainty in context section - -### Pitfall 2: Giant ADRs Combining Many Decisions - -**Symptom**: ADR covers multiple unrelated systems or decisions (e.g., "Database, Message Queue, and API Gateway Strategy v3"). - -**Why It's a Problem**: -- Hard to find specific decisions -- Difficult to supersede individual parts -- Violates single-responsibility principle - -**How to Prevent**: -- Split ADRs by decision, not by system -- Use "Related ADRs" section to link related decisions -- If decisions are tightly coupled, combine only if they form a single coherent pattern - -### Pitfall 3: ADRs with No Recorded Alternatives - -**Symptom**: ADR states the decision but doesn't document what alternatives were considered. - -**Why It's a Problem**: -- No understanding of trade-offs -- Future engineers can't evaluate if decision is still valid -- Missed opportunity to learn from rejected options - -**How to Prevent**: -- Always document at least 2-3 alternatives -- Even if one option is "obvious," document why others were rejected -- Use "Options Considered" section to force explicit comparison - -### Pitfall 4: Never Marking ADRs as Superseded or Deprecated - -**Symptom**: Old ADRs remain "Accepted" even when decisions have changed. - -**Why It's a Problem**: -- Confusion about which decision is current -- New engineers follow outdated guidance -- No clear history of decision evolution - -**How to Prevent**: -- Quarterly ADR review meetings -- When creating new ADR, check if it supersedes an old one -- Update old ADR status to "Superseded" with link to new ADR -- Use "Related ADRs" section to maintain decision lineage - -### Pitfall 5: ADRs That Aren't Linked to Implementation - -**Symptom**: ADR exists but no code, config, or documentation references it. - -**Why It's a Problem**: -- ADR becomes "zombie document" (exists but unused) -- Decision may not actually be followed -- No way to verify implementation - -**How to Prevent**: -- Link ADRs from best-practices documents -- Reference ADRs in design docs and RFCs -- Include ADR numbers in PR descriptions when implementing decisions -- Use ADR review to identify unlinked ADRs - -### Pitfall 6: Vague or Incomplete Context - -**Symptom**: ADR context section is too brief or doesn't explain the problem. - -**Why It's a Problem**: -- Future readers can't understand why decision was made -- Can't evaluate if decision is still relevant -- Missing information leads to poor decisions - -**How to Prevent**: -- Context section should be 2-4 paragraphs -- Explain the problem, constraints, and business/technical drivers -- Include relevant background (e.g., "We previously used X, but...") -- Review context section in ADR review process - -## Implementation Roadmap - -### Phase 1: Seed (Weeks 1-2) - -**Goal**: Establish ADR practice with initial examples. - -**Tasks**: -1. **Create ADR directory structure**: - ```bash - mkdir -p docs/adr/_templates - ``` - -2. **Add templates**: - - Copy standard ADR template to `docs/adr/_templates/0001-template-standard.md` - - Copy lightweight ADR template to `docs/adr/_templates/0002-template-lightweight.md` - -3. **Write 2-3 retrospective ADRs**: - - Document already-decided major items: - - Observability stack (Grafana + Prometheus + Loki) - - Database auditing (PgAudit + PgCron) - - SBOM/Trivy policy - - Use "Accepted" status - - Include as much context as you remember - -4. **Create ADR index**: - - Manually create `docs/adr/index.md` with table of ADRs - - Link to ADR best practices document - -**Deliverables**: -- ADR directory with templates -- 2-3 initial ADRs -- ADR index page - -### Phase 2: Integrate (Weeks 3-4) - -**Goal**: Make ADRs part of standard workflow. - -**Tasks**: -1. **Update contribution guidelines**: - - Add section on "When to Write an ADR" - - Include link to ADR best practices - - Add ADR checklist to PR template - -2. **Require ADR for new major systems**: - - When proposing new service/component, require ADR - - Gate PRs on ADR review for architectural changes - -3. **Link ADRs from best-practices docs**: - - Add ADR references to relevant best-practices documents - - Update existing docs to reference ADRs where applicable - -4. **Train team**: - - Share ADR best practices document - - Walk through examples in team meeting - - Answer questions and gather feedback - -**Deliverables**: -- Updated contribution guidelines -- ADR references in best-practices docs -- Team trained on ADR process - -### Phase 3: Normalize (Months 2-3) - -**Goal**: Make ADRs a natural part of decision-making. - -**Tasks**: -1. **Quarterly ADR reviews**: - - Schedule first review meeting - - Review all ADRs for stale status - - Update superseded/deprecated ADRs - -2. **CI checks on ADR structure**: - - Add pre-commit hook or CI check - - Validate ADR naming and required fields - - Fail PRs with invalid ADRs - -3. **Cross-links from best-practices**: - - Ensure all major best-practices docs reference relevant ADRs - - Add "Related ADRs" sections where missing - -4. **Refine process**: - - Gather feedback from team - - Update ADR templates if needed - - Adjust governance model based on experience - -**Deliverables**: -- Quarterly review process established -- CI validation in place -- Complete cross-linking - -### Phase 4: Mature (Months 4+) - -**Goal**: ADRs are fully integrated and self-sustaining. - -**Tasks**: -1. **Auto-generate ADR index**: - - Create script to generate index from ADR files - - Add to CI/CD or pre-commit hook - - Keep index up-to-date automatically - -2. **Searchable decision registry**: - - Extract ADR metadata (optional) - - Create searchable index or dashboard - - Enable filtering by status, tags, date - -3. **Integration into onboarding**: - - Add ADR section to onboarding docs - - Point new engineers to ADR index - - Explain how to find "why" decisions were made - -4. **Continuous improvement**: - - Regular feedback collection - - Refine templates and process - - Share learnings with other teams - -**Deliverables**: -- Auto-generated ADR index -- Searchable decision registry (optional) -- ADRs in onboarding process - -## Summary - -### Key Takeaways - -1. **ADRs capture "why"** alongside "how" (best-practices docs) -2. **One decision per ADR** keeps them focused and maintainable -3. **Git-based workflow** integrates ADRs into existing PR/review process -4. **Quarterly reviews** prevent stale ADRs and maintain decision lineage -5. **Cross-linking** connects ADRs to implementation guides and runbooks - -### Adoption Checklist - -- [ ] Create `/docs/adr/` directory with templates -- [ ] Write 2-3 retrospective ADRs for major decisions -- [ ] Create ADR index page -- [ ] Update contribution guidelines -- [ ] Link ADRs from relevant best-practices docs -- [ ] Train team on ADR process -- [ ] Schedule first quarterly review -- [ ] Add CI validation for ADR structure -- [ ] Integrate ADRs into onboarding - -### Next Steps - -1. **Start with Phase 1**: Create directory, add templates, write initial ADRs -2. **Get feedback**: Share with team and iterate on process -3. **Normalize**: Make ADRs part of standard workflow -4. **Mature**: Automate and integrate into broader documentation system - -**Remember**: ADRs are a tool, not a goal. The goal is better decision-making and knowledge sharing. Start simple, iterate based on feedback, and focus on value over process. - -## See Also - -- **[Architecture & Design Best Practices](index.md)** - Overview of architecture patterns -- **[Documentation Best Practices](documentation.md)** - General documentation guidelines -- **[Git Workflows & Collaboration](../../best-practices/git/git-workflows-collaboration.md)** - Version control patterns - ---- - -*This guide provides a complete framework for Architecture Decision Records. Start with Phase 1, gather feedback, and iterate. The goal is better decisions, not perfect documentation.* - diff --git a/docs/best-practices/architecture-design/api-gateway-architecture.md b/docs/best-practices/architecture-design/api-gateway-architecture.md index a3d5507..baa5533 100644 --- a/docs/best-practices/architecture-design/api-gateway-architecture.md +++ b/docs/best-practices/architecture-design/api-gateway-architecture.md @@ -1,6 +1,5 @@ # API Gateway Architecture: Best Practices -**Objective**: Establish comprehensive API gateway architecture patterns for routing, security, rate limiting, and API management. When you need API gateway design, when you want unified API entry points, when you need API management—this guide provides the complete framework. ## Introduction diff --git a/docs/best-practices/architecture-design/api-governance-interface-stability.md b/docs/best-practices/architecture-design/api-governance-interface-stability.md index 58c2448..5bbb1be 100644 --- a/docs/best-practices/architecture-design/api-governance-interface-stability.md +++ b/docs/best-practices/architecture-design/api-governance-interface-stability.md @@ -1,6 +1,5 @@ # API Governance, Backward Compatibility Rules, and Cross-Language Interface Stability: Best Practices -**Objective**: Establish comprehensive API governance that ensures backward compatibility, interface stability, and consistent patterns across Python, Go, Rust, and Postgres APIs. When you need API versioning, when you want interface stability, when you need cross-language coherence—this guide provides the complete framework. ## Introduction diff --git a/docs/best-practices/architecture-design/architecture-fitness-functions-governance.md b/docs/best-practices/architecture-design/architecture-fitness-functions-governance.md index 33d6d73..f3507d5 100644 --- a/docs/best-practices/architecture-design/architecture-fitness-functions-governance.md +++ b/docs/best-practices/architecture-design/architecture-fitness-functions-governance.md @@ -1,6 +1,5 @@ # Architectural Fitness Functions and Governance: Measuring and Evolving Architecture Quality -**Objective**: Master production-grade architectural fitness functions and governance across distributed systems, databases, ML pipelines, and polyglot microservices. When you need to measure, evaluate, monitor, and evolve architecture quality—this guide provides complete patterns and implementations. ## Introduction @@ -2036,7 +2035,6 @@ class LLMScorecardGenerator: ## See Also -- **[ADR and Technical Decision Governance](adr-decision-governance.md)** - Decision recording - **[Repository Standardization](repository-standardization-and-governance.md)** - Repository governance - **[System Resilience](../operations-monitoring/system-resilience-and-concurrency.md)** - Resilience patterns diff --git a/docs/best-practices/architecture-design/cache-topology-architecture.md b/docs/best-practices/architecture-design/cache-topology-architecture.md index 97c9787..a57f540 100644 --- a/docs/best-practices/architecture-design/cache-topology-architecture.md +++ b/docs/best-practices/architecture-design/cache-topology-architecture.md @@ -1,6 +1,5 @@ # Cache-Topology Architecture: Best Practices -**Objective**: Establish comprehensive multi-tier cache topology patterns that optimize performance, reduce latency, and manage cache hierarchies across edge, application, and data layers. When you need cache topology, when you want multi-tier caching, when you need cache strategy—this guide provides the complete framework. ## Introduction diff --git a/docs/best-practices/architecture-design/caching-performance.md b/docs/best-practices/architecture-design/caching-performance.md index 47b8c38..32d48aa 100644 --- a/docs/best-practices/architecture-design/caching-performance.md +++ b/docs/best-practices/architecture-design/caching-performance.md @@ -1,6 +1,5 @@ # Caching & Performance Best Practices -**Objective**: Caches are accelerants. Done right, they make pipelines scream. Done wrong, they give you stale lies. Caches are accelerants. Done right, they make pipelines scream. Done wrong, they give you stale lies. @@ -902,4 +901,3 @@ Caching requires understanding performance characteristics, invalidation strateg --- -*This guide provides the complete machinery for caching and performance. The patterns scale from simple database queries to complex geospatial computations, from basic API responses to advanced ML model inference.* diff --git a/docs/best-practices/architecture-design/capacity-planning-and-workload-modeling.md b/docs/best-practices/architecture-design/capacity-planning-and-workload-modeling.md index 550dab7..f00d8ad 100644 --- a/docs/best-practices/architecture-design/capacity-planning-and-workload-modeling.md +++ b/docs/best-practices/architecture-design/capacity-planning-and-workload-modeling.md @@ -1,6 +1,5 @@ # Holistic Capacity Planning, Scaling Economics, and Workload Modeling: Best Practices -**Objective**: Establish comprehensive capacity planning frameworks that model workloads, predict resource needs, and optimize scaling economics across CPU/GPU clusters, data systems, and distributed services. When you need to plan capacity, when you want to model workloads, when you need scaling economics—this guide provides the complete framework. ## Introduction diff --git a/docs/best-practices/architecture-design/ci-cd-pipelines.md b/docs/best-practices/architecture-design/ci-cd-pipelines.md index 583b9cc..c665d0a 100644 --- a/docs/best-practices/architecture-design/ci-cd-pipelines.md +++ b/docs/best-practices/architecture-design/ci-cd-pipelines.md @@ -1,6 +1,5 @@ # Testing & CI/CD Pipelines Best Practices (2025 Edition) -**Objective**: A broken pipeline is silent chaos. Here's how to build pipelines that are fast, reproducible, and hard to kill. A broken pipeline is silent chaos. Here's how to build pipelines that are fast, reproducible, and hard to kill. @@ -992,4 +991,3 @@ CI/CD pipelines require understanding testing strategies, deployment patterns, a --- -*This guide provides the complete machinery for CI/CD pipelines. The patterns scale from simple unit tests to complex multi-environment deployments, from basic automation to advanced GitOps workflows.* diff --git a/docs/best-practices/architecture-design/cognitive-load-developer-experience.md b/docs/best-practices/architecture-design/cognitive-load-developer-experience.md index cda5b9c..8d8b9fc 100644 --- a/docs/best-practices/architecture-design/cognitive-load-developer-experience.md +++ b/docs/best-practices/architecture-design/cognitive-load-developer-experience.md @@ -1,6 +1,5 @@ # Cognitive Load Management and Developer Experience Architecture: Best Practices for Complex Technical Ecosystems -**Objective**: Master production-grade cognitive load management and developer experience patterns across distributed systems, polyglot microservices, and complex technical ecosystems. When you need to minimize cognitive load, maximize developer experience, and enable effective reasoning about architecture—this guide provides complete patterns and implementations. ## Introduction @@ -2217,8 +2216,6 @@ class CognitiveRiskReview: - **[Architectural Fitness Functions](architecture-fitness-functions-governance.md)** - Architecture measurement - **[Repository Standardization](repository-standardization-and-governance.md)** - Repository governance -- **[ADR and Technical Decision Governance](adr-decision-governance.md)** - Decision recording - --- *This guide provides a complete framework for cognitive load management and developer experience. Start with measuring cognitive load, implement DX patterns, reduce complexity, and continuously improve. The goal is systems that remain mentally tractable and enable effective reasoning about architecture.* diff --git a/docs/best-practices/architecture-design/cost-aware-architecture-and-efficiency-governance.md b/docs/best-practices/architecture-design/cost-aware-architecture-and-efficiency-governance.md index a98a67c..2372da1 100644 --- a/docs/best-practices/architecture-design/cost-aware-architecture-and-efficiency-governance.md +++ b/docs/best-practices/architecture-design/cost-aware-architecture-and-efficiency-governance.md @@ -1,6 +1,5 @@ # Cost-Aware Architecture & Resource-Efficiency Governance: Best Practices -**Objective**: Establish comprehensive cost governance frameworks that measure, optimize, and control resource costs across Kubernetes clusters, data systems, ML pipelines, and geospatial workloads. When you need to optimize costs, when you want to rightsize resources, when you need capacity planning—this guide provides the complete framework. ## Introduction diff --git a/docs/best-practices/architecture-design/data-mesh-architecture.md b/docs/best-practices/architecture-design/data-mesh-architecture.md index 8a068ec..9160fae 100644 --- a/docs/best-practices/architecture-design/data-mesh-architecture.md +++ b/docs/best-practices/architecture-design/data-mesh-architecture.md @@ -1,6 +1,5 @@ # Data Mesh Architecture: Best Practices -**Objective**: Establish comprehensive data mesh architecture that enables domain-driven data ownership, decentralized data products, and federated governance. When you need domain ownership, when you want decentralized data, when you need federated governance—this guide provides the complete framework. ## Introduction diff --git a/docs/best-practices/architecture-design/documentation.md b/docs/best-practices/architecture-design/documentation.md index 2fd3acd..d936fdf 100644 --- a/docs/best-practices/architecture-design/documentation.md +++ b/docs/best-practices/architecture-design/documentation.md @@ -1,6 +1,5 @@ # Documentation Best Practices (2025 Edition) -**Objective**: Docs aren't optional—they're survival kits. Without them, your brilliant pipelines collapse into archaeology. Docs aren't optional—they're survival kits. Without them, your brilliant pipelines collapse into archaeology. @@ -1183,4 +1182,3 @@ Documentation requires understanding information architecture, automation patter --- -*This guide provides the complete machinery for documentation. The patterns scale from simple README files to complex multi-site documentation systems, from basic automation to advanced CI/CD integration.* diff --git a/docs/best-practices/architecture-design/environment-config-governance.md b/docs/best-practices/architecture-design/environment-config-governance.md index 714af90..56db1b8 100644 --- a/docs/best-practices/architecture-design/environment-config-governance.md +++ b/docs/best-practices/architecture-design/environment-config-governance.md @@ -1,6 +1,5 @@ # Cross-Environment Configuration Strategy, Drift Prevention, and Multi-Cluster State Management: Best Practices -**Objective**: Master production-grade configuration governance across multiple environments (dev, test, staging, prod, air-gapped, HPC, RKE2, edge). When you need to prevent drift, ensure consistency, and manage configuration lifecycle—this guide provides complete patterns and implementations. ## Introduction diff --git a/docs/best-practices/architecture-design/event-driven-architecture.md b/docs/best-practices/architecture-design/event-driven-architecture.md index 0cc966e..fe173b1 100644 --- a/docs/best-practices/architecture-design/event-driven-architecture.md +++ b/docs/best-practices/architecture-design/event-driven-architecture.md @@ -1,6 +1,5 @@ # Event-Driven Architecture: Design, Deployment, and Hardening Best Practices -**Objective**: Master production-grade event-driven architecture across Kubernetes, messaging systems, databases, and distributed services. When you need to build resilient, scalable event-driven systems with proper observability and security—this guide provides complete patterns and implementations. ## Introduction diff --git a/docs/best-practices/architecture-design/index.md b/docs/best-practices/architecture-design/index.md index a925e4b..a12c134 100644 --- a/docs/best-practices/architecture-design/index.md +++ b/docs/best-practices/architecture-design/index.md @@ -1,6 +1,5 @@ # Architecture & Design Best Practices -**Objective**: Master senior-level architecture and design patterns for production systems. When you need to build robust, scalable architectures, when you want to follow proven methodologies, when you need enterprise-grade patterns—these best practices become your weapon of choice. ## System Architecture @@ -42,8 +41,6 @@ ## Documentation & Governance - **[Documentation Best Practices](documentation.md)** - Writing, organizing, and sustaining useful docs for data + devops projects -- **[ADR and Technical Decision Governance](adr-decision-governance.md)** - Architecture Decision Records (ADRs) and decision governance framework - ## Data Serialization - **[Protocol Buffers with Python](protobuf-python.md)** - Production-ready data serialization and microservices communication @@ -55,4 +52,3 @@ --- -*These best practices provide the complete machinery for building production-ready architectures. Each guide includes architectural patterns, configuration examples, and real-world implementation strategies for enterprise deployment.* diff --git a/docs/best-practices/architecture-design/multi-cloud-federation-portability.md b/docs/best-practices/architecture-design/multi-cloud-federation-portability.md index d4414bb..3b52b5f 100644 --- a/docs/best-practices/architecture-design/multi-cloud-federation-portability.md +++ b/docs/best-practices/architecture-design/multi-cloud-federation-portability.md @@ -1,6 +1,5 @@ # Multi-Cloud Federation & Portability Architecture: Best Practices -**Objective**: Establish comprehensive multi-cloud federation and portability patterns that enable workload portability, vendor independence, and unified operations across AWS, GCP, Azure, and on-premises infrastructure. When you need cloud portability, when you want vendor independence, when you need unified multi-cloud operations—this guide provides the complete framework. ## Introduction diff --git a/docs/best-practices/architecture-design/multi-region-dr-strategy.md b/docs/best-practices/architecture-design/multi-region-dr-strategy.md index d408b09..ac7a5a7 100644 --- a/docs/best-practices/architecture-design/multi-region-dr-strategy.md +++ b/docs/best-practices/architecture-design/multi-region-dr-strategy.md @@ -1,6 +1,5 @@ # Multi-Region, Multi-Cluster Disaster Recovery, Failover Topologies, and Data Sovereignty: Best Practices -**Objective**: Establish comprehensive disaster recovery strategies across multiple regions, clusters, and cloud providers. When you need cross-region failover, when you want data sovereignty compliance, when you need air-gapped DR—this guide provides the complete framework. ## Introduction diff --git a/docs/best-practices/architecture-design/polyglot-interoperability-design.md b/docs/best-practices/architecture-design/polyglot-interoperability-design.md index 45a72af..3e8e0e8 100644 --- a/docs/best-practices/architecture-design/polyglot-interoperability-design.md +++ b/docs/best-practices/architecture-design/polyglot-interoperability-design.md @@ -1,6 +1,5 @@ # Polyglot Interoperability Design: Best Practices -**Objective**: Establish comprehensive polyglot interoperability patterns that enable seamless integration between Python, Go, Rust, Postgres, and other systems. When you need cross-language integration, when you want polyglot systems, when you need interoperability—this guide provides the complete framework. ## Introduction diff --git a/docs/best-practices/architecture-design/protobuf-python.md b/docs/best-practices/architecture-design/protobuf-python.md index 5754a62..6744811 100644 --- a/docs/best-practices/architecture-design/protobuf-python.md +++ b/docs/best-practices/architecture-design/protobuf-python.md @@ -1,6 +1,5 @@ # Protocol Buffers with Python: Production-Ready Data Serialization -**Objective**: Master Protocol Buffers (protobufs) for efficient, type-safe data serialization in Python applications. When you need high-performance data exchange, when you want type-safe APIs, when you're building microservices that communicate efficiently—protobufs become your weapon of choice. Protocol Buffers provide the foundation for efficient data serialization and type-safe communication. Without proper understanding of schema design, code generation, and serialization patterns, you're building inefficient systems that miss the power of binary serialization and schema evolution. This guide shows you how to wield protobufs with the precision of a senior Python engineer. @@ -1374,4 +1373,3 @@ Protocol Buffers provide the foundation for efficient, type-safe data serializat --- -*This guide provides the complete machinery for mastering Protocol Buffers with Python. The patterns scale from simple message serialization to complex microservices communication, from basic schemas to advanced production deployment.* diff --git a/docs/best-practices/architecture-design/rdf-owl-metadata-automation.md b/docs/best-practices/architecture-design/rdf-owl-metadata-automation.md index abee9d8..3d0ccd0 100644 --- a/docs/best-practices/architecture-design/rdf-owl-metadata-automation.md +++ b/docs/best-practices/architecture-design/rdf-owl-metadata-automation.md @@ -1,6 +1,5 @@ # RDF/OWL Metadata Automation: Dynamic Knowledge Graphs with Friend-of-a-Friend Associations -**Objective**: Master RDF/OWL for automated metadata handling and dynamic ontological associations. When you need to automatically discover relationships between entities, when you're building intelligent knowledge graphs, when you want to leverage semantic reasoning for metadata enrichment—RDF/OWL becomes your weapon of choice. RDF/OWL metadata automation is the foundation of intelligent knowledge management. Without proper understanding of semantic web technologies, you're building static, disconnected metadata systems that miss the power of automated reasoning and relationship discovery. This guide shows you how to wield RDF/OWL with the precision of a semantic web engineer. @@ -888,4 +887,3 @@ RDF/OWL metadata automation provides the foundation for intelligent knowledge ma --- -*This guide provides the complete machinery for mastering RDF/OWL metadata automation. The patterns scale from simple ontologies to complex knowledge graphs, from basic reasoning to advanced inference.* diff --git a/docs/best-practices/architecture-design/reference-architecture-diagrams.md b/docs/best-practices/architecture-design/reference-architecture-diagrams.md index 56a32fd..46b3b16 100644 --- a/docs/best-practices/architecture-design/reference-architecture-diagrams.md +++ b/docs/best-practices/architecture-design/reference-architecture-diagrams.md @@ -1,6 +1,5 @@ # Reference Architecture Diagrams: Best Practices -**Objective**: Establish comprehensive reference architecture diagrams that document system topologies, component relationships, and architectural patterns. When you need reference architectures, when you want system documentation, when you need architectural blueprints—this guide provides the complete framework. ## Introduction @@ -23,7 +22,6 @@ Reference architecture diagrams are essential for understanding, communicating, **Related Documents**: This document integrates with: -- **[ADR and Technical Decision Governance](adr-decision-governance.md)** - Decision documentation - **[Documentation](documentation.md)** - Documentation patterns - **[System-Wide Naming, Taxonomy, and Structural Vocabulary Governance](system-taxonomy-governance.md)** - Naming standards @@ -98,7 +96,6 @@ diagram_standards: ## See Also -- **[ADR and Technical Decision Governance](adr-decision-governance.md)** - Decisions - **[Documentation](documentation.md)** - Documentation - **[System-Wide Naming, Taxonomy, and Structural Vocabulary Governance](system-taxonomy-governance.md)** - Naming diff --git a/docs/best-practices/architecture-design/repository-standardization-and-governance.md b/docs/best-practices/architecture-design/repository-standardization-and-governance.md index e031e9d..efac884 100644 --- a/docs/best-practices/architecture-design/repository-standardization-and-governance.md +++ b/docs/best-practices/architecture-design/repository-standardization-and-governance.md @@ -1,6 +1,5 @@ # Repository Standardization, Templates, and Lifecycle Governance: Best Practices for Polyglot Ecosystems -**Objective**: Master production-grade repository standardization across Python, Go, Rust, Docker, Kubernetes, and data pipelines. When you need to eliminate snowflake repos, ensure consistency, and enable automated governance—this guide provides complete patterns and implementations. ## Introduction @@ -2171,7 +2170,6 @@ class LLMDocsIngester: ## See Also -- **[ADR and Technical Decision Governance](adr-decision-governance.md)** - Decision recording - **[Configuration Management](../operations-monitoring/configuration-management.md)** - Config governance - **[Release Management](../operations-monitoring/release-management-and-progressive-delivery.md)** - Deployment practices diff --git a/docs/best-practices/architecture-design/secrets-management.md b/docs/best-practices/architecture-design/secrets-management.md index b7cff6d..108d299 100644 --- a/docs/best-practices/architecture-design/secrets-management.md +++ b/docs/best-practices/architecture-design/secrets-management.md @@ -1,6 +1,5 @@ # Secrets Management Best Practices (2025 Edition) -**Objective**: Secrets are dangerous. They leak, they rot, they get copied into Slack and Git commits. Here's how to lock them down without breaking your developer flow. Secrets are dangerous. They leak, they rot, they get copied into Slack and Git commits. Here's how to lock them down without breaking your developer flow. @@ -799,4 +798,3 @@ Secrets management requires understanding security risks, access patterns, and l --- -*This guide provides the complete machinery for secrets management. The patterns scale from simple environment variables to complex enterprise secret stores, from basic security to advanced threat protection.* diff --git a/docs/best-practices/architecture-design/service-decomposition-strategy.md b/docs/best-practices/architecture-design/service-decomposition-strategy.md index 70cea8a..81eb99f 100644 --- a/docs/best-practices/architecture-design/service-decomposition-strategy.md +++ b/docs/best-practices/architecture-design/service-decomposition-strategy.md @@ -1,6 +1,5 @@ # Service Decomposition Strategy: Best Practices -**Objective**: Establish comprehensive service decomposition strategies that guide when and how to decompose monolithic systems into microservices, bounded contexts, and domain services. When you need decomposition guidance, when you want domain boundaries, when you need service boundaries—this guide provides the complete framework. ## Introduction diff --git a/docs/best-practices/architecture-design/streaming-architecture-patterns.md b/docs/best-practices/architecture-design/streaming-architecture-patterns.md index c591e33..b66e38f 100644 --- a/docs/best-practices/architecture-design/streaming-architecture-patterns.md +++ b/docs/best-practices/architecture-design/streaming-architecture-patterns.md @@ -1,6 +1,5 @@ # Streaming Architecture Patterns: SAGA, CQRS, and Outbox: Best Practices -**Objective**: Establish comprehensive streaming architecture patterns including SAGA for distributed transactions, CQRS for read/write separation, and Outbox for reliable event publishing. When you need distributed transactions, when you want read/write separation, when you need reliable events—this guide provides the complete framework. ## Introduction diff --git a/docs/best-practices/architecture-design/system-taxonomy-governance.md b/docs/best-practices/architecture-design/system-taxonomy-governance.md index 015e99d..a5a60aa 100644 --- a/docs/best-practices/architecture-design/system-taxonomy-governance.md +++ b/docs/best-practices/architecture-design/system-taxonomy-governance.md @@ -1,6 +1,5 @@ # System-Wide Naming, Taxonomy, and Structural Vocabulary Governance: Best Practices -**Objective**: Establish enterprise-wide naming conventions, domain taxonomy, and structural vocabulary that serve as the "lingua franca" across all systems, services, databases, and codebases. When you need consistent naming, when you want to reduce cognitive load, when you need cross-system clarity—this guide provides the complete framework. ## Introduction diff --git a/docs/best-practices/architecture-design/temporal-governance-and-time-synchronization.md b/docs/best-practices/architecture-design/temporal-governance-and-time-synchronization.md index ccca81e..4ca9e3c 100644 --- a/docs/best-practices/architecture-design/temporal-governance-and-time-synchronization.md +++ b/docs/best-practices/architecture-design/temporal-governance-and-time-synchronization.md @@ -1,6 +1,5 @@ # Temporal Governance and Time Synchronization: Best Practices for Distributed Systems -**Objective**: Master production-grade time synchronization across Kubernetes, databases, ML pipelines, and distributed systems. When you need to ensure causality, prevent clock drift, and maintain temporal consistency—this guide provides complete patterns and implementations. ## Introduction diff --git a/docs/best-practices/architecture/cost-aware-systems.md b/docs/best-practices/architecture/cost-aware-systems.md index f64105b..c154827 100644 --- a/docs/best-practices/architecture/cost-aware-systems.md +++ b/docs/best-practices/architecture/cost-aware-systems.md @@ -1,6 +1,5 @@ # Cost-Aware System Architecture -**Objective**: Design systems that account for operational cost as an architectural constraint, not an afterthought. ## Cost as an architectural constraint diff --git a/docs/best-practices/creative-fun/celery-best-practices.md b/docs/best-practices/creative-fun/celery-best-practices.md index ad27dff..ce6259b 100644 --- a/docs/best-practices/creative-fun/celery-best-practices.md +++ b/docs/best-practices/creative-fun/celery-best-practices.md @@ -1,6 +1,5 @@ # Celery in Production: Picking the Right Jobs, Writing Safe Tasks, Running It Like You Mean It -**Objective**: Master Celery task queues for reliable distributed processing in Python applications. When you need to offload HTTP requests, when you want to process background jobs, when you're building scalable data pipelines—Celery becomes your weapon of choice. Celery is a distributed task queue that provides reliable at-least-once execution with retries, scheduling, and monitoring. It's not a streaming processor or OLAP engine—it's designed for offloading work from web requests, processing scheduled jobs, and handling idempotent side effects at scale. @@ -989,4 +988,3 @@ Celery requires understanding both distributed systems patterns and Python concu --- -*This guide provides the complete machinery for Celery in production. The patterns scale from simple HTTP offloading to complex distributed workflows, from basic task execution to advanced orchestration.* diff --git a/docs/best-practices/creative-fun/idempotency-and-dedup.md b/docs/best-practices/creative-fun/idempotency-and-dedup.md index f3dae0e..0d8ba30 100644 --- a/docs/best-practices/creative-fun/idempotency-and-dedup.md +++ b/docs/best-practices/creative-fun/idempotency-and-dedup.md @@ -1,6 +1,5 @@ # Idempotency & De-dup: Designing Operations That Don't Double-Fire -**Objective**: Master idempotency and de-duplication patterns to prevent duplicate operations in distributed systems. When you need safe retries, when you want to prevent double-billing, when you're building reliable APIs and data pipelines—idempotency becomes your weapon of choice. Retries happen (clients, proxies, workers). Without idempotency, you print money twice—or delete the same row twice. Build request identity, dedupe state, and committed outcomes into every layer. @@ -584,4 +583,3 @@ Idempotency requires understanding both system reliability and data consistency --- -*This guide provides the complete machinery for idempotency and de-duplication. The patterns scale from simple HTTP APIs to complex distributed systems, from basic retry handling to advanced stream processing.* diff --git a/docs/best-practices/creative-fun/index.md b/docs/best-practices/creative-fun/index.md index 628a3f4..1098973 100644 --- a/docs/best-practices/creative-fun/index.md +++ b/docs/best-practices/creative-fun/index.md @@ -1,6 +1,5 @@ # Creative & Fun Best Practices -**Objective**: Master creative and fun patterns for production systems. When you need to build engaging, creative solutions, when you want to follow proven methodologies, when you need enterprise-grade patterns—these best practices become your weapon of choice. ## Opinions @@ -15,4 +14,3 @@ --- -*These best practices provide the complete machinery for building production-ready creative systems. Each guide includes architectural patterns, configuration examples, and real-world implementation strategies for enterprise deployment.* diff --git a/docs/best-practices/creative-fun/latex.md b/docs/best-practices/creative-fun/latex.md index a699122..f4db6b9 100644 --- a/docs/best-practices/creative-fun/latex.md +++ b/docs/best-practices/creative-fun/latex.md @@ -1,6 +1,5 @@ # LaTeX Workflows Best Practices -**Objective**: Master production-grade LaTeX workflows for reproducible technical writing, academic papers, and engineering documentation. When you need to write professional technical documents, when you want to create beautiful diagrams and figures, when you're responsible for maintaining consistent formatting across teams—LaTeX workflows become your weapon of choice. LaTeX isn't just typesetting—it's reproducible writing. Done poorly, it's chaos. Done well, it's a publishing pipeline as solid as any codebase. @@ -631,4 +630,3 @@ LaTeX workflows require understanding document structure, build reproducibility, --- -*This guide provides the complete machinery for LaTeX workflows. The patterns scale from simple documents to complex academic papers, from basic formatting to advanced document generation.* diff --git a/docs/best-practices/creative-fun/time-hygiene.md b/docs/best-practices/creative-fun/time-hygiene.md index b9455c6..9645d9d 100644 --- a/docs/best-practices/creative-fun/time-hygiene.md +++ b/docs/best-practices/creative-fun/time-hygiene.md @@ -1,6 +1,5 @@ # Time Hygiene: UTC Everywhere, Monotonic Durations, and DST-Proof Scheduling -**Objective**: Master temporal hygiene for production systems that handle time correctly. When you need reliable scheduling, when you want to prevent DST disasters, when you're building data pipelines that span timezones—time hygiene becomes your weapon of choice. Time is the most misunderstood aspect of system design. Proper time handling prevents DST disasters, enables reliable scheduling, and maintains data integrity across timezones. This guide shows you how to wield time with the precision of a paranoid SRE, covering everything from UTC storage to monotonic clocks and DST-proof scheduling. @@ -806,4 +805,3 @@ Time handling requires understanding both temporal mechanics and system design p --- -*This guide provides the complete machinery for time hygiene. The patterns scale from simple timestamp storage to complex distributed systems, from basic scheduling to advanced temporal data processing.* diff --git a/docs/best-practices/creative-fun/yaml-recipe-format.md b/docs/best-practices/creative-fun/yaml-recipe-format.md index aa02585..9d80414 100644 --- a/docs/best-practices/creative-fun/yaml-recipe-format.md +++ b/docs/best-practices/creative-fun/yaml-recipe-format.md @@ -1,6 +1,5 @@ # YAML Recipe Format: Why Your Kitchen Needs Indentation -**Objective**: Master YAML as the superior format for culinary operations. When you need to version control your grandmother's secret sauce, when you want to lint your pancake recipes, when you're building a CI/CD pipeline for your chili—YAML recipe format becomes your weapon of choice. Other people use index cards. We use declarative configs. @@ -1014,4 +1013,3 @@ YAML recipe format provides the foundation for modern culinary operations. When --- -*This guide provides the complete machinery for mastering YAML recipe format. The patterns scale from simple breakfast recipes to complex multi-course meals, from basic cooking to advanced culinary operations.* diff --git a/docs/best-practices/data-governance/data-freshness-sla-governance.md b/docs/best-practices/data-governance/data-freshness-sla-governance.md index ec459e0..1a90854 100644 --- a/docs/best-practices/data-governance/data-freshness-sla-governance.md +++ b/docs/best-practices/data-governance/data-freshness-sla-governance.md @@ -1,6 +1,5 @@ # Data Freshness, SLA/SLO Governance, and Pipeline Reliability Contracts: Best Practices -**Objective**: Establish comprehensive data freshness governance with SLA/SLO frameworks for ETL pipelines, real-time streaming, geospatial processing, and data serving layers. When you need to ensure data freshness, when you want to define reliability contracts, when you need pipeline SLOs—this guide provides the complete framework. ## Introduction diff --git a/docs/best-practices/data-governance/data-lineage-contracts.md b/docs/best-practices/data-governance/data-lineage-contracts.md index 5a88c48..5ca715a 100644 --- a/docs/best-practices/data-governance/data-lineage-contracts.md +++ b/docs/best-practices/data-governance/data-lineage-contracts.md @@ -1,6 +1,5 @@ # Cross-System Data Lineage, Inter-Service Metadata Contracts & Provenance Enforcement -**Objective**: Establish data lineage and contract enforcement across pipelines, services, and storage so that provenance is traceable, schema and semantics are agreed, and violations are detectable. When you need to trace data from source to consumption, enforce contracts between producers and consumers, or prove compliance—this guide provides the patterns and integration points. ## Introduction diff --git a/docs/best-practices/data-governance/data-quality-sla-validation-observability.md b/docs/best-practices/data-governance/data-quality-sla-validation-observability.md index dc30b72..bdf552f 100644 --- a/docs/best-practices/data-governance/data-quality-sla-validation-observability.md +++ b/docs/best-practices/data-governance/data-quality-sla-validation-observability.md @@ -1,6 +1,5 @@ # Data Quality SLAs, Validation Layers, and Observability for Tabular, Geospatial, and ML Data: Best Practices -**Objective**: Establish comprehensive data quality governance with SLAs, multi-layer validation, and observability for tabular, geospatial, and ML data. When you need data quality assurance, when you want quality SLAs, when you need validation observability—this guide provides the complete framework. ## Introduction diff --git a/docs/best-practices/data-governance/data-retention-archival-lifecycle-governance.md b/docs/best-practices/data-governance/data-retention-archival-lifecycle-governance.md index 76039f7..0754ff2 100644 --- a/docs/best-practices/data-governance/data-retention-archival-lifecycle-governance.md +++ b/docs/best-practices/data-governance/data-retention-archival-lifecycle-governance.md @@ -1,6 +1,5 @@ # Data Retention, Archival Strategy, Lifecycle Governance & Cold Storage Patterns: Best Practices -**Objective**: Establish comprehensive data retention and archival strategies that govern data lifecycle from hot to frozen storage, ensuring compliance, cost optimization, and operational efficiency. When you need retention policies, when you want archival strategies, when you need lifecycle governance—this guide provides the complete framework. ## Introduction diff --git a/docs/best-practices/data-governance/data-validation-and-contract-governance.md b/docs/best-practices/data-governance/data-validation-and-contract-governance.md index de30bee..ccebe35 100644 --- a/docs/best-practices/data-governance/data-validation-and-contract-governance.md +++ b/docs/best-practices/data-governance/data-validation-and-contract-governance.md @@ -1,6 +1,5 @@ # Data Validation and Contract Governance: Best Practices for Polyglot Distributed Systems -**Objective**: Master production-grade data validation and contract governance across Postgres, DuckDB, MLflow, Parquet, ETL pipelines, and distributed systems. When you need to ensure data quality, prevent silent corruption, and maintain contract consistency—this guide provides complete patterns and implementations. ## Introduction diff --git a/docs/best-practices/data-governance/index.md b/docs/best-practices/data-governance/index.md index ae18139..0246cd5 100644 --- a/docs/best-practices/data-governance/index.md +++ b/docs/best-practices/data-governance/index.md @@ -1,6 +1,5 @@ # Data Governance Best Practices -**Objective**: Master production-grade data governance for distributed analytics systems. When you need to ensure data quality, track lineage, enforce contracts, and maintain reproducibility across Postgres, Parquet, MLflow, and ETL pipelines—these best practices become your foundation. This collection provides comprehensive guides for metadata management, schema governance, data provenance, and data contracts. Each guide includes architectural patterns, configuration examples, and real-world implementation strategies. diff --git a/docs/best-practices/data-governance/metadata-provenance-contracts.md b/docs/best-practices/data-governance/metadata-provenance-contracts.md index 0037de2..d286ad3 100644 --- a/docs/best-practices/data-governance/metadata-provenance-contracts.md +++ b/docs/best-practices/data-governance/metadata-provenance-contracts.md @@ -1,6 +1,5 @@ # Metadata Standards, Schema Governance & Data Provenance Contracts: Best Practices for Distributed Analytics Systems -**Objective**: Master production-grade metadata management, schema governance, and data provenance for distributed analytics ecosystems. When you need to ensure data quality, track lineage, enforce contracts, and maintain reproducibility across Postgres, Parquet, MLflow, and ETL pipelines—this guide provides the complete framework. ## Introduction diff --git a/docs/best-practices/data-processing/spark/scaling-spark.md b/docs/best-practices/data-processing/spark/scaling-spark.md index 857f481..cbeea49 100644 --- a/docs/best-practices/data-processing/spark/scaling-spark.md +++ b/docs/best-practices/data-processing/spark/scaling-spark.md @@ -1,6 +1,5 @@ # Scaling Spark Clusters Correctly -**Objective**: Cluster sizing, executor strategy, and partitioning so Spark jobs run efficiently without waste or bottlenecks. ## Spark architecture overview diff --git a/docs/best-practices/data-processing/spark/spark-modern-architecture.md b/docs/best-practices/data-processing/spark/spark-modern-architecture.md index 889ef47..4e2a941 100644 --- a/docs/best-practices/data-processing/spark/spark-modern-architecture.md +++ b/docs/best-practices/data-processing/spark/spark-modern-architecture.md @@ -1,6 +1,5 @@ # Spark in Modern Data Architectures -**Objective**: Place Spark in context among lakehouse, object storage, and modern analytics tools so you can choose and integrate it correctly. ## Historical role of Spark diff --git a/docs/best-practices/data-processing/spark/spark-on-kubernetes.md b/docs/best-practices/data-processing/spark/spark-on-kubernetes.md index a4c8179..03707a9 100644 --- a/docs/best-practices/data-processing/spark/spark-on-kubernetes.md +++ b/docs/best-practices/data-processing/spark/spark-on-kubernetes.md @@ -1,6 +1,5 @@ # Running Spark on Kubernetes -**Objective**: Operational best practices for running Apache Spark on Kubernetes: architecture, deployment modes, resources, and storage. ## Why Kubernetes for Spark diff --git a/docs/best-practices/data-processing/spark/spark-performance-tuning.md b/docs/best-practices/data-processing/spark/spark-performance-tuning.md index c2b18cf..81f3dba 100644 --- a/docs/best-practices/data-processing/spark/spark-performance-tuning.md +++ b/docs/best-practices/data-processing/spark/spark-performance-tuning.md @@ -1,6 +1,5 @@ # Spark Performance Tuning -**Objective**: Tune Spark jobs and data layout for throughput and stability: data layout, execution plans, memory, and join strategies. ## Data layout matters diff --git a/docs/best-practices/data-processing/spark/when-to-use-spark.md b/docs/best-practices/data-processing/spark/when-to-use-spark.md index cb4b042..d476f1c 100644 --- a/docs/best-practices/data-processing/spark/when-to-use-spark.md +++ b/docs/best-practices/data-processing/spark/when-to-use-spark.md @@ -1,6 +1,5 @@ # When to Use Apache Spark (and When Not To) -**Objective**: Help engineers decide when Spark is the right tool and when alternatives are a better fit. ## What Spark is designed for diff --git a/docs/best-practices/data/metadata-control-plane.md b/docs/best-practices/data/metadata-control-plane.md index 6dd4d0e..e944324 100644 --- a/docs/best-practices/data/metadata-control-plane.md +++ b/docs/best-practices/data/metadata-control-plane.md @@ -1,6 +1,5 @@ # Metadata as a Control Plane -**Objective**: Explain how metadata governs data systems and how to treat it as infrastructure rather than documentation. ## Metadata vs data diff --git a/docs/best-practices/data/reproducible-data-pipelines.md b/docs/best-practices/data/reproducible-data-pipelines.md index c008efc..0d8f0f7 100644 --- a/docs/best-practices/data/reproducible-data-pipelines.md +++ b/docs/best-practices/data/reproducible-data-pipelines.md @@ -1,6 +1,5 @@ # Reproducible Data Pipelines -**Objective**: Define practices for building deterministic, auditable data pipelines that produce the same outputs from the same inputs and support governance and debugging. ## Why reproducibility matters @@ -119,6 +118,6 @@ Avoid pipelines that depend on undocumented inputs, unversioned code, or overwri - [ETL Pipeline Design](../database-data/etl-pipeline-design.md) — production ETL patterns and orchestration - [Parquet](../database-data/parquet.md) and [GeoParquet](../database-data/geoparquet.md) — format and layout best practices - [Data Lineage, Contracts & Provenance](../database-data/data-lineage-contracts.md) — lineage and provenance enforcement -- [Data Pipeline Quality, Governance & Observability](../../tutorials/best-practices-integration/data-pipeline-quality-governance-observability.md) — quality and observability in pipelines +- [Metadata as control plane](metadata-control-plane.md) — lineage and operational metadata - [Why Most Data Pipelines Fail](../../deep-dives/why-most-data-pipelines-fail.md) — failure modes and structural fixes - [Lakehouse vs Warehouse vs Database](../../deep-dives/lakehouse-vs-warehouse-vs-database.md) — where pipelines land data diff --git a/docs/best-practices/database-data/ai-ml-geospatial-knowledge-graph.md b/docs/best-practices/database-data/ai-ml-geospatial-knowledge-graph.md index 3259eb6..d7ed10b 100644 --- a/docs/best-practices/database-data/ai-ml-geospatial-knowledge-graph.md +++ b/docs/best-practices/database-data/ai-ml-geospatial-knowledge-graph.md @@ -1,6 +1,5 @@ # Best Practices for Designing an AI-Ready, ML-Enabled Geospatial Knowledge Graph -**Objective**: Establish comprehensive best practices for designing, building, operating, and evolving an AI-ready, ML-enabled Geospatial Knowledge Graph (GeoKG) suitable for production environments, geospatial analytics, and advanced ML/RAG workflows. When you need to combine geospatial data with knowledge graphs, semantic layers, and ML systems—this guide provides the complete framework. ## Abstract diff --git a/docs/best-practices/database-data/aws-serverless-geospatial.md b/docs/best-practices/database-data/aws-serverless-geospatial.md index fcddf0e..633f50c 100644 --- a/docs/best-practices/database-data/aws-serverless-geospatial.md +++ b/docs/best-practices/database-data/aws-serverless-geospatial.md @@ -1,6 +1,5 @@ # AWS Serverless Geospatial Processing -**Objective**: Build scalable, cost-effective geospatial processing pipelines using AWS serverless services. Serverless architecture eliminates infrastructure management while providing automatic scaling and pay-per-use pricing. This guide covers building production-ready geospatial processing systems on AWS. diff --git a/docs/best-practices/database-data/data-lake-governance.md b/docs/best-practices/database-data/data-lake-governance.md index cdb95ad..082bf43 100644 --- a/docs/best-practices/database-data/data-lake-governance.md +++ b/docs/best-practices/database-data/data-lake-governance.md @@ -1,6 +1,5 @@ # Best Practices for Data Lake Governance -**Objective**: Master data lake governance for trustworthy, auditable, compliant data at scale. When you need to enforce data quality, when you're building compliance workflows, when you need to scale to billions of objects—data lake governance becomes your weapon of choice. Data lake governance is the foundation of trustworthy data infrastructure. Without proper governance, data lakes become data swamps with shadow data, schema drift, and compliance violations. This guide shows you how to design and enforce governance with the precision of a data architect who has been burned before. @@ -1003,4 +1002,3 @@ Data lake governance provides the foundation for trustworthy, auditable, complia --- -*This guide provides the complete machinery for mastering data lake governance. The patterns scale from development to production, from simple schemas to enterprise-grade compliance frameworks.* diff --git a/docs/best-practices/database-data/database-migrations.md b/docs/best-practices/database-data/database-migrations.md index c46fb4a..28f0933 100644 --- a/docs/best-practices/database-data/database-migrations.md +++ b/docs/best-practices/database-data/database-migrations.md @@ -1,6 +1,5 @@ # Database Migrations & Schema Evolution -**Objective**: Databases are living systems. Schema changes are inevitable. Handle them without breaking prod, corrupting data, or waking ops in the night. Databases are living systems. Schema changes are inevitable. Handle them without breaking prod, corrupting data, or waking ops in the night. @@ -1002,4 +1001,3 @@ Database migrations require understanding schema evolution, data safety, and pro --- -*This guide provides the complete machinery for database migrations. The patterns scale from simple table changes to complex geospatial schema evolution, from basic Alembic usage to advanced zero-downtime strategies.* diff --git a/docs/best-practices/database-data/etl-pipeline-design.md b/docs/best-practices/database-data/etl-pipeline-design.md index 37361da..ae21c62 100644 --- a/docs/best-practices/database-data/etl-pipeline-design.md +++ b/docs/best-practices/database-data/etl-pipeline-design.md @@ -1,6 +1,5 @@ # ETL Pipeline Design -**Objective**: Build robust, scalable ETL pipelines for geospatial data processing using modern orchestration tools. ETL pipelines are the backbone of data engineering. This guide covers designing production-ready ETL systems that handle geospatial data at scale with reliability and performance. diff --git a/docs/best-practices/database-data/geoparquet-data-warehouses.md b/docs/best-practices/database-data/geoparquet-data-warehouses.md index 8777199..fc0b935 100644 --- a/docs/best-practices/database-data/geoparquet-data-warehouses.md +++ b/docs/best-practices/database-data/geoparquet-data-warehouses.md @@ -1,6 +1,5 @@ # GeoParquet Data Warehouses -**Objective**: Build modern geospatial data warehouses using GeoParquet format for optimal performance and interoperability. GeoParquet represents the future of geospatial data storage—columnar, compressed, and cross-platform compatible. This guide covers building production-ready geospatial data warehouses that scale. diff --git a/docs/best-practices/database-data/geospatial-benchmarking.md b/docs/best-practices/database-data/geospatial-benchmarking.md index 5fb8713..0b9b1df 100644 --- a/docs/best-practices/database-data/geospatial-benchmarking.md +++ b/docs/best-practices/database-data/geospatial-benchmarking.md @@ -1,6 +1,5 @@ # Best Practices for Geospatial Benchmarking under CPU/GPU Stress -**Objective**: Master geospatial benchmarking methodologies for high-performance computing environments. When you need to stress-test spatial algorithms, when you're optimizing for GPU acceleration, when you need to validate performance under load—geospatial benchmarking becomes your weapon of choice. Geospatial benchmarking is the foundation of performance validation for spatial computing. Without proper benchmarking, you're flying blind into production with algorithms that may fail under load, scale poorly, or consume excessive resources. This guide shows you how to design and execute comprehensive geospatial benchmarks with the precision of an HPC engineer. @@ -1052,4 +1051,3 @@ Geospatial benchmarking under CPU/GPU stress provides the foundation for perform --- -*This guide provides the complete machinery for mastering geospatial benchmarking under CPU/GPU stress. The patterns scale from development to production, from simple operations to enterprise-grade spatial computing systems.* diff --git a/docs/best-practices/database-data/index.md b/docs/best-practices/database-data/index.md index 8174dab..f986171 100644 --- a/docs/best-practices/database-data/index.md +++ b/docs/best-practices/database-data/index.md @@ -1,6 +1,5 @@ # Database & Data Management Best Practices -**Objective**: Master senior-level database and data management patterns for production systems. When you need to build robust, scalable data systems, when you want to follow proven methodologies, when you need enterprise-grade patterns—these best practices become your weapon of choice. ## PostgreSQL & High Availability @@ -36,4 +35,3 @@ --- -*These best practices provide the complete machinery for building production-ready data systems. Each guide includes architectural patterns, configuration examples, and real-world implementation strategies for enterprise deployment.* diff --git a/docs/best-practices/database-data/lake-vs-lakehouse-vs-warehouse.md b/docs/best-practices/database-data/lake-vs-lakehouse-vs-warehouse.md index 32ed32d..a70a2ec 100644 --- a/docs/best-practices/database-data/lake-vs-lakehouse-vs-warehouse.md +++ b/docs/best-practices/database-data/lake-vs-lakehouse-vs-warehouse.md @@ -1,6 +1,5 @@ # Choosing Data Lakes, Lakehouses, and Warehouses: Trade-offs, Patterns, and Free Stacks -**Objective**: Master the architectural trade-offs between data lakes, lakehouses, and warehouses for modern data infrastructure. When you need to choose the right data architecture, when you're building scalable analytics platforms, when you need to balance cost, performance, and governance—data architecture becomes your weapon of choice. Data architecture decisions determine the fate of your analytics platform. Without proper understanding of the trade-offs, you'll build expensive, slow, or ungovernable systems. This guide shows you how to choose and implement the right architecture with the precision of a principal data architect. @@ -951,4 +950,3 @@ Data architecture decisions determine the fate of your analytics platform. When --- -*This guide provides the complete machinery for choosing and implementing data architectures. The patterns scale from development to production, from simple analytics to enterprise-grade data platforms.* diff --git a/docs/best-practices/database-data/patroni-postgres-ha.md b/docs/best-practices/database-data/patroni-postgres-ha.md index 3c1e0a9..4c79322 100644 --- a/docs/best-practices/database-data/patroni-postgres-ha.md +++ b/docs/best-practices/database-data/patroni-postgres-ha.md @@ -1,6 +1,5 @@ # Patroni PostgreSQL HA: The Art of High Availability -**Objective**: Master Patroni to build bulletproof PostgreSQL clusters that survive hardware failures, network partitions, and catastrophic disasters. When your database becomes the single point of failure, when downtime costs thousands per minute, when data loss is not an option—Patroni becomes your weapon of choice. PostgreSQL high availability is the bridge between single-node databases and enterprise-grade resilience. Without proper HA setup, you're flying blind into production with databases that could fail in ways that destroy your business. This guide shows you how to wield Patroni with the precision of a seasoned database engineer. @@ -910,4 +909,3 @@ Patroni PostgreSQL HA is the foundation of reliable database deployments. When c --- -*This tutorial provides the complete machinery for mastering Patroni PostgreSQL HA. The patterns scale from development to production, from simple clusters to enterprise-grade deployments.* diff --git a/docs/best-practices/database-data/semantic-layer-engineering.md b/docs/best-practices/database-data/semantic-layer-engineering.md index 4e5cb31..5b3481a 100644 --- a/docs/best-practices/database-data/semantic-layer-engineering.md +++ b/docs/best-practices/database-data/semantic-layer-engineering.md @@ -1,6 +1,5 @@ # Semantic Layer Engineering, Domain Models, and Knowledge Graph Alignment: Best Practices -**Objective**: Establish enterprise semantic layers that bridge business concepts to physical storage across geospatial, infrastructure, ML/AI, and data domains. When you need domain models, when you want knowledge graph alignment, when you need semantic versioning—this guide provides the complete framework. ## Introduction diff --git a/docs/best-practices/diagrams/svg-workflow-generation.md b/docs/best-practices/diagrams/svg-workflow-generation.md index 09ea658..504e518 100644 --- a/docs/best-practices/diagrams/svg-workflow-generation.md +++ b/docs/best-practices/diagrams/svg-workflow-generation.md @@ -231,7 +231,7 @@ A future GitHub Actions workflow might look like: ## Example: Data Platform Workflow -The following Mermaid source represents the full data platform workflow for this site — sensors through serving, with governance and observability cross-cuts. +The following Mermaid source is a full data platform workflow — sensors through serving, with governance and observability cross-cuts. **Mermaid source** (`docs/assets/diagrams/workflows/data-platform-workflow.mmd`): @@ -316,6 +316,6 @@ flowchart LR --- !!! tip "See also" - - [Mermaid → SVG Workflow Pipeline (Tutorial)](../../tutorials/diagrams/mermaid-to-svg-workflow-pipeline.md) — step-by-step guide to setting up and running the render pipeline - - [Diagram Style Guide](../../diagrams/style-guide.md) — Mermaid snippet library and formatting conventions - - [ADR 0015: Standardize Diagrams on Mermaid](../../adr/0015-diagrams-mermaid.md) — the architectural decision record that established this standard + - [Rendering Mermaid diagrams to SVG](../../tutorials/diagrams/mermaid-to-svg-workflow-pipeline.md) — step-by-step setup and render pipeline + - [Systems Diagramming Best Practices](systems-diagramming-best-practices.md) — layering, boundaries, and entropy control + - [Layered Systems Diagrams tutorial](../../tutorials/diagrams/layered-systems-diagrams-mermaid-to-svg.md) — end-to-end IoT → lakehouse example diff --git a/docs/best-practices/diagrams/systems-diagramming-best-practices.md b/docs/best-practices/diagrams/systems-diagramming-best-practices.md index cb06e1e..61c70b5 100644 --- a/docs/best-practices/diagrams/systems-diagramming-best-practices.md +++ b/docs/best-practices/diagrams/systems-diagramming-best-practices.md @@ -9,7 +9,6 @@ tags: # Systems Diagramming Best Practices (Mermaid, SVG, and Diagram Discipline) -**Objective**: Define a clear doctrine for systems diagramming: conceptual vs physical diagrams, layering, system boundary clarity, and diagram entropy control so that diagrams remain accurate, readable, and maintainable. --- @@ -110,13 +109,13 @@ The diagram makes the trust boundary explicit: Internet → DMZ → Internal. Us | Fast iteration and co-editing in docs | Reuse in other tools; consistent layout | | Diagram lives in the repo with the doc | You need a single export for external use | -### Repo convention +### Mermaid source and SVG artifacts -- **Source**: `.mmd` files under `docs/assets/diagrams/` (e.g. `examples/iot-mqtt-lakehouse-context.mmd`). -- **Artifact**: `.svg` committed **beside** the `.mmd` (same directory, same basename). -- **Render**: Run the site’s render script from `tools/diagrams/` (e.g. `npm run render:all`) so SVG is regenerated from source. Never edit the SVG by hand; it will be overwritten. +- **Source**: version-controlled `.mmd` files (one diagram per file). +- **Artifact**: `.svg` committed beside the `.mmd` (same directory, same basename) when static docs must render without a Mermaid runtime. +- **Render**: regenerate SVG from source with `@mermaid-js/mermaid-cli` or an equivalent script. Do not hand-edit generated SVG; it will be overwritten on the next render. -See [Generating Complex Workflow Diagrams as SVG](svg-workflow-generation.md) and the [Layered Systems Diagrams tutorial](../../tutorials/diagrams/layered-systems-diagrams-mermaid-to-svg.md) for the full pipeline. +See [Generating Complex Workflow Diagrams as SVG](svg-workflow-generation.md) and the [Layered Systems Diagrams tutorial](../../tutorials/diagrams/layered-systems-diagrams-mermaid-to-svg.md) for a worked example. --- @@ -153,7 +152,7 @@ See [Generating Complex Workflow Diagrams as SVG](svg-workflow-generation.md) an - **Proprietary formats vs committed SVG**: Prefer committing SVG (and, when possible, Mermaid source) so the doc build and future readers don’t depend on a SaaS tool. - **Collaboration vs reproducibility**: For one-off workshops, Lucid or draw.io may be faster. For long-lived docs, Mermaid + rendered SVG gives reproducibility and version control. -This site standardizes on **Mermaid as source** and **SVG as artifact** via `tools/diagrams/`. See the [Mermaid → SVG Workflow Pipeline](../../tutorials/diagrams/mermaid-to-svg-workflow-pipeline.md) and the [Diagram Style Guide](../../diagrams/style-guide.md). +For documentation that ships as static HTML, **Mermaid as source** and **committed SVG as artifact** is a practical default. See the [Mermaid → SVG workflow tutorial](../../tutorials/diagrams/mermaid-to-svg-workflow-pipeline.md) for setup and rendering steps. --- @@ -284,7 +283,7 @@ flowchart LR F -.-> H[Observability] ``` -The full example—source `.mmd` files and rendered SVG artifacts—lives under `docs/assets/diagrams/examples/` and is used in the [Layered Systems Diagrams: Mermaid Source → SVG Artifact](../../tutorials/diagrams/layered-systems-diagrams-mermaid-to-svg.md) tutorial. Follow that tutorial to generate the SVGs and embed them in MkDocs. +A full IoT → MQTT → lakehouse example—with `.mmd` sources and rendered SVG—is in the [Layered Systems Diagrams: Mermaid → SVG](../../tutorials/diagrams/layered-systems-diagrams-mermaid-to-svg.md) tutorial. --- @@ -293,7 +292,6 @@ The full example—source `.mmd` files and rendered SVG artifacts—lives under !!! tip "See also" - **[Layered Systems Diagrams: Mermaid Source → SVG Artifact](../../tutorials/diagrams/layered-systems-diagrams-mermaid-to-svg.md)** — End-to-end tutorial with the IoT → MQTT → Lakehouse example and SVG embedding. - - **[Generating Complex Workflow Diagrams as SVG](svg-workflow-generation.md)** — Mermaid-first, artifact-driven workflow and repository conventions. - - **[Mermaid → SVG Workflow Pipeline](../../tutorials/diagrams/mermaid-to-svg-workflow-pipeline.md)** — Existing pipeline for rendering `.mmd` to `.svg` with `tools/diagrams/`. - - **[Diagram Style Guide](../../diagrams/style-guide.md)** — When to use Mermaid vs SVG, orientation, subgraphs, and accessibility. + - **[Generating Complex Workflow Diagrams as SVG](svg-workflow-generation.md)** — Mermaid-first, artifact-driven workflow and layout conventions. + - **[Rendering Mermaid diagrams to SVG](../../tutorials/diagrams/mermaid-to-svg-workflow-pipeline.md)** — Set up `mmdc` and render `.mmd` to `.svg`. - **Deep dives**: [Observability vs Monitoring](../../deep-dives/observability-vs-monitoring.md), [Why Most Data Pipelines Fail](../../deep-dives/why-most-data-pipelines-fail.md) — Systems context that diagrams should reflect. diff --git a/docs/best-practices/docker-infrastructure/ansible-inventory-management.md b/docs/best-practices/docker-infrastructure/ansible-inventory-management.md index 71a6950..3c12c85 100644 --- a/docs/best-practices/docker-infrastructure/ansible-inventory-management.md +++ b/docs/best-practices/docker-infrastructure/ansible-inventory-management.md @@ -1,6 +1,5 @@ # Ansible Inventory Management Best Practices -**Objective**: Master Ansible inventory design for scalable, maintainable automation. When you need to manage complex infrastructure, when you want consistent configuration across environments, when you're building enterprise automation—inventory management becomes your weapon of choice. Ansible inventory is the foundation of automation. Proper inventory design enables scalable deployments, environment separation, and maintainable configuration. This guide shows you how to wield inventory management with the precision of a DevOps engineer. @@ -639,4 +638,3 @@ Ansible inventory management requires understanding both infrastructure patterns --- -*This guide provides the complete machinery for Ansible inventory management. The patterns scale from simple single-environment setups to complex multi-environment deployments, from basic host management to advanced dynamic inventory and secrets management.* diff --git a/docs/best-practices/docker-infrastructure/ansible-performance-optimization.md b/docs/best-practices/docker-infrastructure/ansible-performance-optimization.md index 874b9e8..82184a8 100644 --- a/docs/best-practices/docker-infrastructure/ansible-performance-optimization.md +++ b/docs/best-practices/docker-infrastructure/ansible-performance-optimization.md @@ -1,6 +1,5 @@ # Ansible Performance Optimization Best Practices -**Objective**: Master Ansible performance patterns for enterprise-scale automation. When you need to optimize automation workflows, when you want to reduce execution time and resource usage, when you're building high-performance infrastructure—performance optimization becomes your weapon of choice. Ansible performance is critical for enterprise automation. Proper optimization enables faster deployments, reduced resource usage, and scalable automation. This guide shows you how to wield performance optimization with the precision of a systems engineer. @@ -569,4 +568,3 @@ Ansible performance optimization requires understanding both automation patterns --- -*This guide provides the complete machinery for Ansible performance optimization. The patterns scale from basic task optimization to advanced enterprise scaling, from simple caching to complex resource management.* diff --git a/docs/best-practices/docker-infrastructure/ansible-playbook-design.md b/docs/best-practices/docker-infrastructure/ansible-playbook-design.md index b9d6358..1012453 100644 --- a/docs/best-practices/docker-infrastructure/ansible-playbook-design.md +++ b/docs/best-practices/docker-infrastructure/ansible-playbook-design.md @@ -1,6 +1,5 @@ # Ansible Playbook Design Best Practices -**Objective**: Master Ansible playbook architecture for maintainable, scalable automation. When you need to build complex automation workflows, when you want reusable and testable code, when you're creating enterprise-grade automation—playbook design becomes your weapon of choice. Ansible playbooks are the heart of automation. Proper playbook design enables maintainable code, reusable components, and scalable automation. This guide shows you how to wield playbook design with the precision of a DevOps engineer. @@ -685,4 +684,3 @@ Ansible playbook design requires understanding both automation patterns and soft --- -*This guide provides the complete machinery for Ansible playbook design. The patterns scale from simple single-task playbooks to complex multi-role deployments, from basic automation to advanced enterprise patterns.* diff --git a/docs/best-practices/docker-infrastructure/ansible-security-hardening.md b/docs/best-practices/docker-infrastructure/ansible-security-hardening.md index 05d8cd4..3303682 100644 --- a/docs/best-practices/docker-infrastructure/ansible-security-hardening.md +++ b/docs/best-practices/docker-infrastructure/ansible-security-hardening.md @@ -1,6 +1,5 @@ # Ansible Security Hardening Best Practices -**Objective**: Master Ansible security patterns for enterprise-grade automation. When you need to secure automation workflows, when you want to protect sensitive data and credentials, when you're building compliant infrastructure—security hardening becomes your weapon of choice. Ansible security is critical for enterprise automation. Proper security practices prevent credential exposure, ensure compliance, and protect sensitive infrastructure. This guide shows you how to wield security hardening with the precision of a security engineer. @@ -609,4 +608,3 @@ Ansible security hardening requires understanding both automation patterns and s --- -*This guide provides the complete machinery for Ansible security hardening. The patterns scale from basic credential protection to advanced enterprise security, from simple access controls to complex compliance frameworks.* diff --git a/docs/best-practices/docker-infrastructure/conda-to-docker-migration.md b/docs/best-practices/docker-infrastructure/conda-to-docker-migration.md index 24bf4a5..00afbc0 100644 --- a/docs/best-practices/docker-infrastructure/conda-to-docker-migration.md +++ b/docs/best-practices/docker-infrastructure/conda-to-docker-migration.md @@ -1,6 +1,5 @@ # Migrating Conda to Slim Docker Images -**Objective**: Transform bloated conda environments into production-ready, slim Docker images. Eliminate conda bloat while maintaining package compatibility and build reproducibility. When your conda environment is dragging gigabytes of unnecessary packages, you're paying for storage and transfer costs with your soul. This guide shows how to migrate from conda to Docker while keeping images lean and production-ready. @@ -693,4 +692,3 @@ Migrating from conda to slim Docker images requires understanding your dependenc --- -*This tutorial provides the complete machinery for migrating conda environments to slim Docker images. The patterns scale from development to production, from megabytes to terabytes.* diff --git a/docs/best-practices/docker-infrastructure/docker-sbom-trivy-cve-mitigation.md b/docs/best-practices/docker-infrastructure/docker-sbom-trivy-cve-mitigation.md index aca7c82..c16c284 100644 --- a/docs/best-practices/docker-infrastructure/docker-sbom-trivy-cve-mitigation.md +++ b/docs/best-practices/docker-infrastructure/docker-sbom-trivy-cve-mitigation.md @@ -1,6 +1,5 @@ # Best Practices for SBOMs, Trivy Scans, and Automated CVE Mitigation in Docker Images -**Objective**: Master production-grade container security with SBOM generation, vulnerability scanning, and automated CVE mitigation. When you need to secure containerized workloads, maintain compliance, and reduce manual security toil—these best practices become your foundation. ## Introduction @@ -1580,5 +1579,4 @@ This approach transforms container security from a manual, error-prone process i --- -*This guide provides the complete machinery for production-grade container security. The patterns scale from single images to enterprise registries, from manual processes to fully automated remediation workflows.* diff --git a/docs/best-practices/docker-infrastructure/index.md b/docs/best-practices/docker-infrastructure/index.md index 9200d01..6b35796 100644 --- a/docs/best-practices/docker-infrastructure/index.md +++ b/docs/best-practices/docker-infrastructure/index.md @@ -1,6 +1,5 @@ # Docker & Infrastructure Best Practices -**Objective**: Master senior-level infrastructure and containerization patterns for production systems. When you need to build robust, scalable infrastructure, when you want to follow proven methodologies, when you need enterprise-grade patterns—these best practices become your weapon of choice. ## Containerization & Orchestration @@ -25,4 +24,3 @@ --- -*These best practices provide the complete machinery for building production-ready infrastructure systems. Each guide includes architectural patterns, configuration examples, and real-world implementation strategies for enterprise deployment.* diff --git a/docs/best-practices/docker-infrastructure/jinja-best-practices.md b/docs/best-practices/docker-infrastructure/jinja-best-practices.md index bbc0d8b..a92b244 100644 --- a/docs/best-practices/docker-infrastructure/jinja-best-practices.md +++ b/docs/best-practices/docker-infrastructure/jinja-best-practices.md @@ -1,6 +1,5 @@ # Jinja Best Practices: Architecture, Power Tricks, and Safety -**Objective**: Master Jinja templating for production-grade text generation. When you need to generate configuration files, documentation, or any structured text from data, when you want maintainable and secure templating, when you're building code generation or documentation systems—Jinja becomes your weapon of choice. Jinja is a compiler for text. Treat it like code: testable, linted, secure, and fast. This guide shows you how to wield Jinja with the precision of a senior template sorcerer, covering everything from basic patterns to advanced security and performance optimization. @@ -830,4 +829,3 @@ Jinja templating requires understanding both template patterns and security impl --- -*This guide provides the complete machinery for Jinja templating. The patterns scale from simple text substitution to complex document generation, from basic security to advanced performance optimization.* diff --git a/docs/best-practices/docker-infrastructure/nginx-best-practices.md b/docs/best-practices/docker-infrastructure/nginx-best-practices.md index 46b127f..1135a8a 100644 --- a/docs/best-practices/docker-infrastructure/nginx-best-practices.md +++ b/docs/best-practices/docker-infrastructure/nginx-best-practices.md @@ -1,6 +1,5 @@ # NGINX Best Practices: Patterns, Hardening, and Multi-API Replication -**Objective**: Master production-grade NGINX configuration for reverse proxying, load balancing, SSL termination, and multi-API gateways. When you need to front multiple services, replicate traffic for testing, and harden your edge layer—these best practices become your foundation. ## Introduction @@ -1763,5 +1762,4 @@ This maturity path takes you from a basic reverse proxy to a production-grade AP --- -*This guide provides the complete machinery for production-grade NGINX configuration. The patterns scale from simple reverse proxies to complex multi-API gateways with traffic mirroring, from single instances to load-balanced clusters.* diff --git a/docs/best-practices/docker-infrastructure/nginx-production.md b/docs/best-practices/docker-infrastructure/nginx-production.md index f597138..8488a0a 100644 --- a/docs/best-practices/docker-infrastructure/nginx-production.md +++ b/docs/best-practices/docker-infrastructure/nginx-production.md @@ -1,6 +1,5 @@ # Nginx Production Best Practices -**Objective**: Build bulletproof web infrastructure with Nginx. Handle traffic spikes, secure your applications, and optimize for performance while maintaining operational sanity. When your web application needs to handle millions of requests, serve static content at lightning speed, and protect against every attack vector imaginable, Nginx becomes your first line of defense. This guide shows you how to configure Nginx for production workloads that demand reliability, security, and performance. @@ -1183,4 +1182,3 @@ Nginx is the foundation of modern web infrastructure. When configured properly, --- -*This tutorial provides the complete machinery for building production-ready Nginx infrastructure. The patterns scale from development to production, from single machines to enterprise deployments.* diff --git a/docs/best-practices/docker-infrastructure/tmux-advanced.md b/docs/best-practices/docker-infrastructure/tmux-advanced.md index 37d034a..f8de532 100644 --- a/docs/best-practices/docker-infrastructure/tmux-advanced.md +++ b/docs/best-practices/docker-infrastructure/tmux-advanced.md @@ -1,6 +1,5 @@ # Advanced Tmux Workflows (2025 Edition) -**Objective**: Master production-grade tmux workflows for distributed geospatial and data engineering environments. When you need to orchestrate complex multi-service workflows, when you want to maintain persistent development environments, when you're debugging across multiple systems—advanced tmux becomes your weapon of choice. tmux is more than pane-splitting—it's your personal mission control. @@ -702,4 +701,3 @@ Advanced tmux requires understanding distributed systems, workflow orchestration --- -*This guide provides the complete machinery for advanced tmux workflows. The patterns scale from simple pane management to complex orchestration, from local development to distributed team collaboration.* diff --git a/docs/best-practices/esp32/index.md b/docs/best-practices/esp32/index.md index d3eeef7..fd52268 100644 --- a/docs/best-practices/esp32/index.md +++ b/docs/best-practices/esp32/index.md @@ -6,7 +6,6 @@ tags: # 🔌 Embedded Systems & ESP32 -**Objective**: Build firmware that is safe, power-efficient, and maintainable. These guides cover the real decisions you make when designing embedded systems around the ESP32 — not just "how to blink an LED" but how to architect, power, secure, and ship firmware responsibly. ## Guides in this section diff --git a/docs/best-practices/geospatial/geospatial-system-design.md b/docs/best-practices/geospatial/geospatial-system-design.md index 74ccfe7..5bdd4ab 100644 --- a/docs/best-practices/geospatial/geospatial-system-design.md +++ b/docs/best-practices/geospatial/geospatial-system-design.md @@ -1,6 +1,5 @@ # Geospatial System Architecture -**Objective**: Establish foundational patterns for designing geospatial data systems: primitives, indexing, pipelines, storage, and performance. ## Core geospatial primitives @@ -84,4 +83,4 @@ For benchmarking and comparison see [Geospatial Benchmarking](../database-data/g - [Reproducible Data Pipelines](../data/reproducible-data-pipelines.md) — determinism in spatial pipelines - [Geospatial File Format Choices](../../deep-dives/geospatial-file-format-choices.md) — format tradeoffs - [The Operational Geometry of Spatial Systems](../../deep-dives/the-operational-geometry-of-spatial-systems.md) — indexing and geometry -- [Geospatial Data Mesh, Cost & Capacity](../../tutorials/best-practices-integration/geospatial-data-mesh-cost-capacity.md) — data mesh and cost in geospatial +- [Cost-aware system architecture](../architecture/cost-aware-systems.md) — cost trade-offs in system design diff --git a/docs/best-practices/git/git-production.md b/docs/best-practices/git/git-production.md index 4cfdbcf..69158f0 100644 --- a/docs/best-practices/git/git-production.md +++ b/docs/best-practices/git/git-production.md @@ -1,6 +1,5 @@ # Git Production Best Practices -**Objective**: Master Git for enterprise-scale development. Handle monorepos, submodules, and complex workflows while maintaining code quality and team collaboration. When your codebase spans multiple applications, shared libraries, and deployment environments, Git becomes more than version control—it becomes the foundation of your development workflow. This guide shows you how to structure repositories, manage dependencies, and maintain code quality at scale. @@ -920,4 +919,3 @@ Git is the foundation of modern development workflows. When configured properly, --- -*This tutorial provides the complete machinery for building production-ready Git workflows. The patterns scale from development to production, from single developers to enterprise teams.* diff --git a/docs/best-practices/git/git-workflows-collaboration.md b/docs/best-practices/git/git-workflows-collaboration.md index b6cc1c0..edc3e09 100644 --- a/docs/best-practices/git/git-workflows-collaboration.md +++ b/docs/best-practices/git/git-workflows-collaboration.md @@ -1,6 +1,5 @@ # Git Workflows and Collaboration Patterns -**Objective**: Master Git workflows for enterprise-grade collaboration and code quality. When you need to coordinate multiple developers, when you want to maintain code quality and project stability, when you're building scalable development processes—Git workflows become your weapon of choice. Git workflows are the foundation of modern software development. Proper workflow design enables seamless collaboration, maintains code quality, and prevents integration nightmares. This guide shows you how to wield Git workflows with the precision of a senior software engineer, covering everything from basic branching to advanced collaboration patterns. @@ -654,4 +653,3 @@ Git workflows require understanding both version control mechanics and team coll --- -*This guide provides the complete machinery for Git workflows and collaboration. The patterns scale from simple feature development to complex enterprise processes, from basic branching to advanced automation and security.* diff --git a/docs/best-practices/git/gitflow-best-practices.md b/docs/best-practices/git/gitflow-best-practices.md index 2ac498e..9e0b8c3 100644 --- a/docs/best-practices/git/gitflow-best-practices.md +++ b/docs/best-practices/git/gitflow-best-practices.md @@ -1,6 +1,5 @@ # Modern GitFlow Best Practices: Branching, Releases, Hotfixes, and CI/CD Governance -**Objective**: Master production-grade GitFlow workflows for enterprise software development. When you need structured branching, controlled releases, hotfix processes, and CI/CD governance—these best practices become your foundation. ## Introduction @@ -1349,5 +1348,4 @@ main (trunk) --- -*This guide provides the complete machinery for production-grade GitFlow workflows. The patterns scale from small teams to enterprise organizations, from simple releases to complex multi-version maintenance.* diff --git a/docs/best-practices/git/index.md b/docs/best-practices/git/index.md index 1c527df..0289ebd 100644 --- a/docs/best-practices/git/index.md +++ b/docs/best-practices/git/index.md @@ -1,6 +1,5 @@ # Git Best Practices -**Objective**: Master Git for enterprise-scale development, collaboration, and code quality. When you need to structure repositories, manage complex workflows, coordinate team collaboration, and maintain code quality at scale—these Git best practices become your foundation. This collection provides comprehensive guides for Git repository management, workflow design, and team collaboration patterns. Each guide includes architectural patterns, configuration examples, and real-world implementation strategies. diff --git a/docs/best-practices/go/go-api-design.md b/docs/best-practices/go/go-api-design.md index d321cd5..65d419f 100644 --- a/docs/best-practices/go/go-api-design.md +++ b/docs/best-practices/go/go-api-design.md @@ -1,6 +1,5 @@ # Go API Design Best Practices -**Objective**: Master senior-level Go API design patterns for production systems. When you need to build robust, scalable RESTful APIs, when you want to follow proven methodologies, when you need enterprise-grade API design patterns—these best practices become your weapon of choice. ## Core Principles @@ -1103,4 +1102,3 @@ func (h *Handler) ListUsers(w http.ResponseWriter, r *http.Request) { --- -*This guide provides the complete machinery for building production-ready RESTful APIs in Go applications. Each pattern includes implementation examples, validation strategies, and real-world usage patterns for enterprise deployment.* diff --git a/docs/best-practices/go/go-cicd-pipelines.md b/docs/best-practices/go/go-cicd-pipelines.md index f215f9a..5f94615 100644 --- a/docs/best-practices/go/go-cicd-pipelines.md +++ b/docs/best-practices/go/go-cicd-pipelines.md @@ -1,6 +1,5 @@ # Go CI/CD Pipelines Best Practices -**Objective**: Master senior-level Go CI/CD pipeline patterns for production systems. When you need to build robust, automated deployment pipelines, when you want to ensure code quality and security, when you need enterprise-grade CI/CD patterns—these best practices become your weapon of choice. ## Core Principles @@ -883,4 +882,3 @@ pipeline { --- -*This guide provides the complete machinery for building robust CI/CD pipelines for Go applications. Each pattern includes implementation examples, security considerations, and real-world usage patterns for enterprise deployment.* diff --git a/docs/best-practices/go/go-concurrency-optimization.md b/docs/best-practices/go/go-concurrency-optimization.md index 83727d4..6ce43e3 100644 --- a/docs/best-practices/go/go-concurrency-optimization.md +++ b/docs/best-practices/go/go-concurrency-optimization.md @@ -1,6 +1,5 @@ # Go Concurrency Optimization Best Practices -**Objective**: Master senior-level Go concurrency optimization patterns for production systems. When you need to build high-performance concurrent applications, when you want to optimize existing concurrent code, when you need enterprise-grade concurrency optimization patterns—these best practices become your weapon of choice. ## Core Principles @@ -1034,4 +1033,3 @@ stats := metrics.GetStats() --- -*This guide provides the complete machinery for optimizing concurrency in Go applications. Each pattern includes implementation examples, performance strategies, and real-world usage patterns for enterprise deployment.* diff --git a/docs/best-practices/go/go-concurrency-patterns.md b/docs/best-practices/go/go-concurrency-patterns.md index 2f31ec3..5c24674 100644 --- a/docs/best-practices/go/go-concurrency-patterns.md +++ b/docs/best-practices/go/go-concurrency-patterns.md @@ -1,6 +1,5 @@ # Go Concurrency Patterns Best Practices -**Objective**: Master senior-level Go concurrency patterns for production systems. When you need to build robust, scalable concurrent applications, when you want to follow proven methodologies, when you need enterprise-grade concurrency patterns—these best practices become your weapon of choice. ## Core Principles @@ -832,4 +831,3 @@ if !limiter.Allow() { --- -*This guide provides the complete machinery for building production-ready concurrent Go applications. Each pattern includes implementation examples, testing strategies, and real-world usage patterns for enterprise deployment.* diff --git a/docs/best-practices/go/go-containerization.md b/docs/best-practices/go/go-containerization.md index 5527429..35c7a3c 100644 --- a/docs/best-practices/go/go-containerization.md +++ b/docs/best-practices/go/go-containerization.md @@ -1,6 +1,5 @@ # Go Containerization Best Practices -**Objective**: Master senior-level Go containerization patterns for production systems. When you need to build efficient, secure containers, when you want to optimize container performance, when you need enterprise-grade containerization patterns—these best practices become your weapon of choice. ## Core Principles @@ -963,4 +962,3 @@ services: --- -*This guide provides the complete machinery for containerizing Go applications efficiently and securely. Each pattern includes implementation examples, security considerations, and real-world usage patterns for enterprise deployment.* diff --git a/docs/best-practices/go/go-data-processing.md b/docs/best-practices/go/go-data-processing.md index 79b89f1..06c403a 100644 --- a/docs/best-practices/go/go-data-processing.md +++ b/docs/best-practices/go/go-data-processing.md @@ -1,6 +1,5 @@ # Go Data Processing Best Practices -**Objective**: Master senior-level Go data processing patterns for production systems. When you need to build efficient data pipelines, when you want to process large datasets, when you need enterprise-grade data processing patterns—these best practices become your weapon of choice. ## Core Principles @@ -1212,4 +1211,3 @@ eventStream.Start() --- -*This guide provides the complete machinery for building efficient data processing pipelines in Go applications. Each pattern includes implementation examples, performance considerations, and real-world usage patterns for enterprise deployment.* diff --git a/docs/best-practices/go/go-database-patterns.md b/docs/best-practices/go/go-database-patterns.md index 4ecc015..b72a9e5 100644 --- a/docs/best-practices/go/go-database-patterns.md +++ b/docs/best-practices/go/go-database-patterns.md @@ -1,6 +1,5 @@ # Go Database Patterns Best Practices -**Objective**: Master senior-level Go database patterns for production systems. When you need to build robust, scalable database applications, when you want to optimize database performance, when you need enterprise-grade database patterns—these best practices become your weapon of choice. ## Core Principles @@ -1246,4 +1245,3 @@ err := orm.Create(ctx, user) --- -*This guide provides the complete machinery for building robust database applications in Go. Each pattern includes implementation examples, performance considerations, and real-world usage patterns for enterprise deployment.* diff --git a/docs/best-practices/go/go-dev-environment.md b/docs/best-practices/go/go-dev-environment.md index 7c07ea5..bd1e66f 100644 --- a/docs/best-practices/go/go-dev-environment.md +++ b/docs/best-practices/go/go-dev-environment.md @@ -1,6 +1,5 @@ # Go Development Environment Best Practices -**Objective**: Master senior-level Go development environment setup and operation across macOS, Linux, and Windows (WSL and native). Copy-paste runnable, auditable, and production-ready. ## Core Principles @@ -720,4 +719,3 @@ make benchmark # Run benchmarks --- -*This guide provides the complete machinery for setting up a production-ready Go development environment. Each pattern includes configuration examples, tooling setup, and real-world implementation strategies for enterprise deployment.* diff --git a/docs/best-practices/go/go-error-handling.md b/docs/best-practices/go/go-error-handling.md index 35e1131..9ac5181 100644 --- a/docs/best-practices/go/go-error-handling.md +++ b/docs/best-practices/go/go-error-handling.md @@ -1,6 +1,5 @@ # Go Error Handling Best Practices -**Objective**: Master senior-level Go error handling patterns for production systems. When you need to build robust, maintainable applications, when you want to follow proven methodologies, when you need enterprise-grade error handling patterns—these best practices become your weapon of choice. ## Core Principles @@ -723,4 +722,3 @@ logger.LogError(err, map[string]interface{}{ --- -*This guide provides the complete machinery for building production-ready error handling in Go applications. Each pattern includes implementation examples, testing strategies, and real-world usage patterns for enterprise deployment.* diff --git a/docs/best-practices/go/go-geospatial-development.md b/docs/best-practices/go/go-geospatial-development.md index 66c280f..c09e51a 100644 --- a/docs/best-practices/go/go-geospatial-development.md +++ b/docs/best-practices/go/go-geospatial-development.md @@ -1,6 +1,5 @@ # Go Geospatial Development Best Practices -**Objective**: Master senior-level Go geospatial development patterns for production systems. When you need to build location-aware applications, when you want to process spatial data efficiently, when you need enterprise-grade geospatial patterns—these best practices become your weapon of choice. ## Core Principles @@ -1134,4 +1133,3 @@ profile, err := profiler.ProfileQuery(ctx, "SELECT * FROM locations WHERE ST_DWi --- -*This guide provides the complete machinery for building geospatial applications in Go. Each pattern includes implementation examples, performance considerations, and real-world usage patterns for enterprise deployment.* diff --git a/docs/best-practices/go/go-memory-management.md b/docs/best-practices/go/go-memory-management.md index d122fa2..dd80848 100644 --- a/docs/best-practices/go/go-memory-management.md +++ b/docs/best-practices/go/go-memory-management.md @@ -1,6 +1,5 @@ # Go Memory Management Best Practices -**Objective**: Master senior-level Go memory management patterns for production systems. When you need to optimize memory usage, when you want to understand Go's garbage collector, when you need enterprise-grade memory management patterns—these best practices become your weapon of choice. ## Core Principles @@ -844,4 +843,3 @@ debug.SetMemoryLimit(512 * 1024 * 1024) // 512MB limit --- -*This guide provides the complete machinery for optimizing memory usage in Go applications. Each pattern includes implementation examples, monitoring strategies, and real-world usage patterns for enterprise deployment.* diff --git a/docs/best-practices/go/go-microservices.md b/docs/best-practices/go/go-microservices.md index ee0cb5c..181f795 100644 --- a/docs/best-practices/go/go-microservices.md +++ b/docs/best-practices/go/go-microservices.md @@ -1,6 +1,5 @@ # Go Microservices Best Practices -**Objective**: Master senior-level Go microservices patterns for production systems. When you need to build robust, scalable distributed systems, when you want to follow proven methodologies, when you need enterprise-grade microservices patterns—these best practices become your weapon of choice. ## Core Principles @@ -1041,4 +1040,3 @@ defer span.Finish() --- -*This guide provides the complete machinery for building production-ready microservices in Go applications. Each pattern includes implementation examples, testing strategies, and real-world usage patterns for enterprise deployment.* diff --git a/docs/best-practices/go/go-monitoring-observability.md b/docs/best-practices/go/go-monitoring-observability.md index 6f2052b..f304567 100644 --- a/docs/best-practices/go/go-monitoring-observability.md +++ b/docs/best-practices/go/go-monitoring-observability.md @@ -1,6 +1,5 @@ # Go Monitoring & Observability Best Practices -**Objective**: Master senior-level Go monitoring and observability patterns for production systems. When you need to build comprehensive monitoring solutions, when you want to implement distributed tracing, when you need enterprise-grade observability patterns—these best practices become your weapon of choice. ## Core Principles @@ -1255,4 +1254,3 @@ alertManager.AddRule(AlertRule{ --- -*This guide provides the complete machinery for implementing comprehensive monitoring and observability in Go applications. Each pattern includes implementation examples, integration strategies, and real-world usage patterns for enterprise deployment.* diff --git a/docs/best-practices/go/go-performance-tuning.md b/docs/best-practices/go/go-performance-tuning.md index ee743a6..754559f 100644 --- a/docs/best-practices/go/go-performance-tuning.md +++ b/docs/best-practices/go/go-performance-tuning.md @@ -1,6 +1,5 @@ # Go Performance Tuning Best Practices -**Objective**: Master senior-level Go performance tuning patterns for production systems. When you need to build high-performance applications, when you want to optimize existing code, when you need enterprise-grade performance patterns—these best practices become your weapon of choice. ## Core Principles @@ -1072,4 +1071,3 @@ metrics.RecordRequest(duration, false) --- -*This guide provides the complete machinery for optimizing Go applications for maximum performance. Each pattern includes implementation examples, benchmarking strategies, and real-world usage patterns for enterprise deployment.* diff --git a/docs/best-practices/go/go-testing-best-practices.md b/docs/best-practices/go/go-testing-best-practices.md index d744466..dba533e 100644 --- a/docs/best-practices/go/go-testing-best-practices.md +++ b/docs/best-practices/go/go-testing-best-practices.md @@ -1,6 +1,5 @@ # Go Testing Best Practices -**Objective**: Master senior-level Go testing patterns for production systems. When you need to build robust, maintainable test suites, when you want to follow proven methodologies, when you need enterprise-grade testing patterns—these best practices become your weapon of choice. ## Core Principles @@ -800,4 +799,3 @@ func BenchmarkPackageName_FunctionName(b *testing.B) { --- -*This guide provides the complete machinery for building production-ready test suites in Go applications. Each pattern includes implementation examples, testing strategies, and real-world usage patterns for enterprise deployment.* diff --git a/docs/best-practices/go/go-web-services.md b/docs/best-practices/go/go-web-services.md index 44f0102..ee5ce7c 100644 --- a/docs/best-practices/go/go-web-services.md +++ b/docs/best-practices/go/go-web-services.md @@ -1,6 +1,5 @@ # Go Web Services Best Practices -**Objective**: Master senior-level Go web service patterns for production systems. When you need to build robust, scalable HTTP services, when you want to follow proven methodologies, when you need enterprise-grade web service patterns—these best practices become your weapon of choice. ## Core Principles @@ -940,4 +939,3 @@ server.RunWithGracefulShutdown() --- -*This guide provides the complete machinery for building production-ready web services in Go applications. Each pattern includes implementation examples, security considerations, and real-world usage patterns for enterprise deployment.* diff --git a/docs/best-practices/go/index.md b/docs/best-practices/go/index.md index de1b9ca..2f84762 100644 --- a/docs/best-practices/go/index.md +++ b/docs/best-practices/go/index.md @@ -1,6 +1,5 @@ # Go Development Best Practices -**Objective**: Master senior-level Go development patterns for production systems. When you need to build robust, scalable Go applications, when you want to follow proven methodologies, when you need enterprise-grade patterns—these best practices become your weapon of choice. ## Core Go Development @@ -36,4 +35,3 @@ --- -*These best practices provide the complete machinery for building production-ready Go systems. Each guide includes architectural patterns, configuration examples, and real-world implementation strategies for enterprise deployment.* diff --git a/docs/best-practices/index.md b/docs/best-practices/index.md index efe9c37..1439d7b 100644 --- a/docs/best-practices/index.md +++ b/docs/best-practices/index.md @@ -1,170 +1,54 @@ # Best Practices -Notes on how systems behave in the wild: trade-offs, failure modes, governance, and the parts of operations that survive contact with production. +Notes on how systems behave in the wild: trade-offs, failure modes, and the parts of operations that survive contact with production. These pages answer *why* and *when* more often than *follow these steps in order*. Code shows up when it clarifies a pattern — the point is judgment, not a runbook. -## 🐍 Python Development +## Start here -Comprehensive guides for Python development, covering core Python, web development, and production patterns. +- [Geospatial system architecture](geospatial/geospatial-system-design.md) +- [Failure-oriented system design](operations/failure-oriented-design.md) +- [System resilience and concurrency](operations-monitoring/system-resilience-and-concurrency.md) +- [PostGIS patterns](postgres/postgis-best-practices.md) +- [Parquet and GeoParquet](database-data/parquet.md) · [GeoParquet](database-data/geoparquet.md) -- **[Python Development Overview](python/index.md)** - Core Python, web development, and production patterns +## Languages and runtimes -## 🦀 Rust Development +- [Python](python/index.md) +- [Rust](rust/index.md) +- [Go](go/index.md) +- [R](r/index.md) -Systems programming and high-performance Rust development patterns. +## Data and platforms -- **[Rust Development Overview](rust/index.md)** - Systems programming and high-performance patterns +- [Database and data management](database-data/index.md) +- [PostgreSQL](postgres/index.md) +- [Machine learning and AI](ml-ai/index.md) +- [Docker and infrastructure](docker-infrastructure/index.md) +- [Git and version control](git/index.md) -## 🐹 Go Development +## Architecture and operations -High-performance systems programming and web services with Go. +- [Architecture and design](architecture-design/index.md) +- [Operations and monitoring](operations-monitoring/index.md) +- [Data governance](data-governance/index.md) +- [Security](security/index.md) -- **[Go Development Overview](go/index.md)** - Systems programming, web services, and performance optimization +## Embedded and home lab -## 📊 R Development +- [ESP32 and embedded](esp32/index.md) +- [Power electronics](embedded/index.md) +- [Home automation](home-automation/index.md) -Statistical computing, data analysis, and reproducible research with R. +## Diagrams -- **[R Development Overview](r/index.md)** - Data science, statistical modeling, and reproducible research +- [Systems diagramming](diagrams/systems-diagramming-best-practices.md) +- [Mermaid → SVG workflow](diagrams/svg-workflow-generation.md) -## 🐳 Docker & Infrastructure +## Creative reference -Containerization, orchestration, and infrastructure automation patterns. - -- **[Docker & Infrastructure Overview](docker-infrastructure/index.md)** - Containerization, orchestration, and automation - -## 📝 Git & Version Control - -Version control, repository management, and collaboration workflows. - -- **[Git & Version Control Overview](git/index.md)** - Repository structure, workflows, and collaboration patterns - -## 🗄️ Database & Data Management - -Database optimization, data architecture, and governance patterns. - -- **[Database & Data Management Overview](database-data/index.md)** - PostgreSQL, data architecture, and governance -- **GeoParquet** — [Spatial Parquet done right](database-data/geoparquet.md): WKB geometry, GeoParquet metadata & CRS rules, object-storage serving (Garage/MinIO/AWS), and `parquet_s3_fdw` wiring with row-group filtering -- **[PostgreSQL Development Overview](postgres/index.md)** - PostgreSQL optimization, performance, and enterprise patterns - -## 🤖 Machine Learning & AI - -ML operations, AI integration, and data science patterns. - -- **[Machine Learning & AI Overview](ml-ai/index.md)** - ML operations, AI integration, and data science - -## 🏗️ Architecture & Design - -System architecture, knowledge management, and data serialization patterns. - -- **[Architecture & Design Overview](architecture-design/index.md)** - System architecture, knowledge management, and serialization - -### 🎯 Integrated Best Practices Suite - -A cohesive suite of five deeply interrelated best-practices documents that form an integrated "constellation" of practices: - -1. **[System-Wide Naming, Taxonomy, and Structural Vocabulary Governance](architecture-design/system-taxonomy-governance.md)** - The foundational "lingua franca" that enables all other practices -2. **[Cross-System Data Lineage, Inter-Service Metadata Contracts & Provenance Enforcement](database-data/data-lineage-contracts.md)** - Comprehensive data lineage and provenance tracking -3. **[Secure-by-Design Lifecycle Architecture Across Polyglot Systems](security/secure-by-design-polyglot.md)** - Security as a lifecycle concern across all languages -4. **[Observability as Architecture: Unified Telemetry Models](operations-monitoring/unified-observability-architecture.md)** - Unified observability across all systems -5. **[Developer Experience (DX) as Infrastructure: Golden Paths, Tooling Ecosystems & Workflow Automation](python/dx-architecture-and-golden-paths.md)** - DX patterns that integrate all practices - -These documents are designed to work together, with each referencing the others and showing how they combine to reduce cognitive load, operational entropy, and system complexity. - -### 🎯 Architectural Resilience & Data Governance Suite - -A complementary suite of four deeply interrelated best-practices documents covering critical architectural gaps: - -1. **[Chaos Engineering, Fault Injection, and Reliability Validation](operations-monitoring/chaos-engineering-governance.md)** - Safe fault injection and systematic reliability validation across all system layers -2. **[Multi-Region, Multi-Cluster Disaster Recovery, Failover Topologies, and Data Sovereignty](architecture-design/multi-region-dr-strategy.md)** - Comprehensive DR strategies with RTO/RPO frameworks and data sovereignty -3. **[Semantic Layer Engineering, Domain Models, and Knowledge Graph Alignment](database-data/semantic-layer-engineering.md)** - Enterprise semantic layers with RDF/OWL integration and entity resolution -4. **[ML Systems Architecture: Feature Stores, Model Serving, Experiment Governance, and Cross-System Reproducibility](ml-ai/ml-systems-architecture-governance.md)** - Complete ML lifecycle architecture with reproducibility and governance - -These documents form a cohesive framework for resilience, data governance, and ML systems, with each document cross-referencing the others and integrating with the foundational taxonomy and observability practices. - -### 🎯 System Efficiency & Data Reliability Suite - -A focused suite of three best-practices documents covering critical operational and governance gaps: - -1. **[Cost-Aware Architecture & Resource-Efficiency Governance](architecture-design/cost-aware-architecture-and-efficiency-governance.md)** - Comprehensive cost governance frameworks for measuring, optimizing, and controlling resource costs across all system layers -2. **[Data Freshness, SLA/SLO Governance, and Pipeline Reliability Contracts](data-governance/data-freshness-sla-governance.md)** - Data freshness governance with SLA/SLO frameworks for ETL pipelines, real-time streaming, and geospatial processing -3. **[Secure Computes, Sandboxing, and Multi-Tenant Isolation for Polyglot Systems](security/secure-sandboxing-and-multi-tenant-isolation.md)** - Comprehensive sandboxing and multi-tenant isolation patterns across all system components - -These documents address critical operational concerns: cost efficiency, data reliability, and secure isolation, with each integrating into the broader best-practices ecosystem. - -### 🎯 Foundational Architecture & Operations Suite - -A critical suite of three best-practices documents covering fundamental architectural and operational gaps: - -1. **[Holistic Capacity Planning, Scaling Economics, and Workload Modeling](architecture-design/capacity-planning-and-workload-modeling.md)** - Systematic capacity planning frameworks for modeling workloads, predicting resource needs, and optimizing scaling economics -2. **[Data Retention, Archival Strategy, Lifecycle Governance & Cold Storage Patterns](data-governance/data-retention-archival-lifecycle-governance.md)** - Comprehensive data retention and archival strategies governing data lifecycle from hot to frozen storage -3. **[Operational Risk Modeling, Blast Radius Reduction & Failure Domain Architecture](operations-monitoring/blast-radius-risk-modeling.md)** - Risk modeling frameworks for identifying failure domains, modeling blast radius, and designing containment strategies - -These documents address foundational concerns: capacity planning, data lifecycle, and operational risk—essential for any mature, coherent technical architecture. - -### 🎯 Enterprise Architecture & Governance Suite - -A comprehensive suite of six best-practices documents covering critical enterprise-grade architectural and operational gaps: - -1. **[Cross-Domain Identity Federation, AuthZ/AuthN Architecture & Identity Propagation Models](security/identity-federation-authz-authn-architecture.md)** - Unified identity federation architecture across all system layers and environments -2. **[Cross-Environment Configuration Drift Prevention, Promotion Workflows & Release Channels](operations-monitoring/environment-promotion-drift-governance.md)** - Controlled environment promotion governance with drift prevention and release channel management -3. **[Data Quality SLAs, Validation Layers, and Observability for Tabular, Geospatial, and ML Data](data-governance/data-quality-sla-validation-observability.md)** - Comprehensive data quality governance with SLAs and multi-layer validation -4. **[API Governance, Backward Compatibility Rules, and Cross-Language Interface Stability](architecture-design/api-governance-interface-stability.md)** - API governance ensuring backward compatibility and interface stability across languages -5. **[Secret Supply Chains, Encryption Lifecycle Management & Cryptographic Rotation Strategy](security/encryption-lifecycle-and-crypto-rotation.md)** - Encryption lifecycle governance with key rotation and cryptographic supply chain management -6. **[Observability-Driven Development (ODD), Telemetry-First Coding Practices, and Preemptive Debugging Architecture](operations-monitoring/observability-driven-development.md)** - Embedding observability as a first-class design input with preemptive debugging patterns - -These documents address enterprise concerns: identity federation, environment promotion, data quality, API governance, encryption lifecycle, and observability-driven development—essential for mature, coherent distributed systems. - -## 🔧 Operations & Monitoring - -Performance monitoring, logging, testing, and security patterns. - -- **[Operations & Monitoring Overview](operations-monitoring/index.md)** - Performance, logging, testing, and security - -## 🔌 Embedded Systems & ESP32 - -Safe, power-efficient, and maintainable firmware patterns for ESP32-based projects. - -- **[Embedded Systems Overview](esp32/index.md)** - Programming architecture, power management, safety, security, sensors, and e-ink displays -- **[Programming Architecture](esp32/esp32-programming-architecture.md)** - Event loops, FreeRTOS tasks, state machines, ISR safety, memory discipline -- **[Power Management & Deep Sleep](esp32/power-management-and-deep-sleep.md)** - Sleep modes, RTC memory, battery safety, sub-100 µA design -- **[Hardware & Electrical Safety](esp32/esp32-hardware-and-electrical-safety.md)** - 3.3 V logic, GPIO limits, level shifting, LiPo safety -- **[Embedded Security & OTA](esp32/embedded-security-and-ota.md)** - NVS secrets, secure boot, OTA updates, WiFi hygiene -- **[Sensor Integration](esp32/sensor-integration-best-practices.md)** - I2C/SPI wiring, ADC caveats, filtering, calibration -- **[E-Ink Display Integration](esp32/e-ink-display-best-practices.md)** - Partial vs full refresh, ghosting, hibernate, frame buffers -- **[MQTT Security](esp32/mqtt-security-best-practices.md)** - Broker hardening, ACLs, TLS, topic design, credential storage for IoT -- **[LoRa Best Practices (SX127x)](esp32/lora-best-practices-sx127x.md)** - Spreading factor, duty cycle compliance, antenna layout, power strategy, AES encryption for raw LoRa -- **[Safety Checklist (Printable)](esp32/esp32-safety-checklist-printable.md)** - Pre-build checklist covering electrical, power, firmware, RF, and physical safety — print to PDF -- **[ESP32-S3 and C3 Notes](esp32/esp32-s3-and-c3-architecture-notes.md)** - USB native, RISC-V, BLE 5.0, variant selection guide, DS/HMAC security peripherals - -## 🔋 Power Electronics & Embedded Hardware - -Cross-platform hardware engineering patterns — applies to ESP32, RP2040, and any microcontroller project. - -- **[Power Electronics Overview](embedded/index.md)** - Section overview -- **[Power Electronics for ESP32](embedded/power-electronics-for-esp32.md)** - Logic-level MOSFETs, high/low-side switching, LiPo charging circuits, solar input, buck vs LDO, brownout protection - -## 🏠 Home Automation & MQTT - -Secure home automation architecture patterns. - -- **[Home Automation Overview](home-automation/index.md)** - Security-focused guides for home automation systems -- **[Home Assistant Security Best Practices](home-automation/home-assistant-security-best-practices.md)** - Access control, TLS, reverse proxy, IoT VLAN, zero-trust mindset - -## 🖼️ Diagrams - -Systems diagramming doctrine, layering, boundaries, and Mermaid → SVG workflow. - -- **[Systems Diagramming Best Practices](diagrams/systems-diagramming-best-practices.md)** — Conceptual vs logical vs physical, layering (L0–L3), boundary clarity, entropy control, pattern library -- **[Generating Complex Workflow Diagrams as SVG](diagrams/svg-workflow-generation.md)** — Mermaid-first, artifact-driven workflow and repository conventions - -## 🎨 Creative & Fun - -Creative solutions, opinions, and fun content patterns. - -- **[Creative & Fun Overview](creative-fun/index.md)** - Creative solutions, opinions, and fun content +- [Creative and fun patterns](creative-fun/index.md) — idempotency, Celery notes, and similar (not the same as [Just for Fun tutorials](../tutorials/just-for-fun/index.md)) --- -*Pick a section that matches the problem; cross-links inside each guide point to related tutorials when you need a hands-on walkthrough.* \ No newline at end of file +*Pick a section that matches the problem; cross-links inside each guide point to [tutorials](../tutorials/index.md) when you need a hands-on walkthrough.* diff --git a/docs/best-practices/ml-ai/embeddings-and-vector-databases.md b/docs/best-practices/ml-ai/embeddings-and-vector-databases.md index 72294f3..7589789 100644 --- a/docs/best-practices/ml-ai/embeddings-and-vector-databases.md +++ b/docs/best-practices/ml-ai/embeddings-and-vector-databases.md @@ -1,6 +1,5 @@ # Embeddings, Vector Databases, and Embedding LLMs — A Survival Guide -**Objective**: Master production-grade embeddings, vector databases, and embedding LLMs for search, RAG, and semantic applications. When you need to build scalable semantic search, when you want to implement production RAG systems, when you're deploying embedding models at scale—vector databases become your weapon of choice. Vector databases provide the foundation for semantic search and retrieval-augmented generation. Without proper understanding of embeddings, vector operations, and production deployment, you're building fragile systems that miss the power of semantic understanding. This guide shows you how to wield vector databases with the precision of a senior search engineer. @@ -1228,4 +1227,3 @@ Vector databases provide the foundation for semantic search and RAG systems. Whe --- -*This guide provides the complete machinery for mastering embeddings, vector databases, and embedding LLMs. The patterns scale from simple similarity search to complex RAG systems, from basic retrieval to advanced semantic understanding.* diff --git a/docs/best-practices/ml-ai/index.md b/docs/best-practices/ml-ai/index.md index 83773ba..3490d87 100644 --- a/docs/best-practices/ml-ai/index.md +++ b/docs/best-practices/ml-ai/index.md @@ -1,6 +1,5 @@ # Machine Learning & AI Best Practices -**Objective**: Master senior-level machine learning and AI patterns for production systems. When you need to build robust, scalable ML systems, when you want to follow proven methodologies, when you need enterprise-grade patterns—these best practices become your weapon of choice. ## ML Systems Architecture @@ -20,4 +19,3 @@ --- -*These best practices provide the complete machinery for building production-ready ML and AI systems. Each guide includes architectural patterns, configuration examples, and real-world implementation strategies for enterprise deployment.* diff --git a/docs/best-practices/ml-ai/mcp-fastapi-stack.md b/docs/best-practices/ml-ai/mcp-fastapi-stack.md index 121296b..34b5814 100644 --- a/docs/best-practices/ml-ai/mcp-fastapi-stack.md +++ b/docs/best-practices/ml-ai/mcp-fastapi-stack.md @@ -1,6 +1,5 @@ # Python MCP Done Right: Tools, Router, and LLM Integration with FastAPI -**Objective**: Master Model Context Protocol (MCP) for building secure, scalable tool servers with intelligent routing and LLM integration. When you need to expose tools to LLMs safely, when you want to build policy-driven tool orchestration, when you're creating production-ready AI toolchains—MCP becomes your weapon of choice. MCP provides the foundation for secure, scalable tool integration with LLMs. Without proper understanding of protocol design, security boundaries, and orchestration patterns, you're building vulnerable systems that miss the power of controlled tool execution. This guide shows you how to wield MCP with the precision of a senior systems engineer. @@ -965,4 +964,3 @@ MCP provides the foundation for secure, scalable AI tool integration. When used --- -*This guide provides the complete machinery for mastering MCP with Python and FastAPI. The patterns scale from simple tool servers to complex AI orchestration systems, from basic security to advanced production deployment.* diff --git a/docs/best-practices/ml-ai/ml-systems-architecture-governance.md b/docs/best-practices/ml-ai/ml-systems-architecture-governance.md index 88c4b23..706c0cf 100644 --- a/docs/best-practices/ml-ai/ml-systems-architecture-governance.md +++ b/docs/best-practices/ml-ai/ml-systems-architecture-governance.md @@ -1,6 +1,5 @@ # ML Systems Architecture: Feature Stores, Model Serving, Experiment Governance, and Cross-System Reproducibility -**Objective**: Establish comprehensive ML systems architecture covering the full lifecycle from data ingestion to model deployment and monitoring. When you need feature stores, when you want experiment governance, when you need reproducibility—this guide provides the complete framework. ## Introduction diff --git a/docs/best-practices/ml-ai/onnx-model-optimization.md b/docs/best-practices/ml-ai/onnx-model-optimization.md index c4469ab..4dd818e 100644 --- a/docs/best-practices/ml-ai/onnx-model-optimization.md +++ b/docs/best-practices/ml-ai/onnx-model-optimization.md @@ -1,6 +1,5 @@ # ONNX Model Optimization: Production-Ready Machine Learning Deployment -**Objective**: Master ONNX model creation, optimization, and deployment for production machine learning systems. When you need cross-platform model deployment, when you're optimizing for inference performance, when you need to standardize model formats across frameworks—ONNX becomes your weapon of choice. ONNX model optimization is the foundation of production machine learning deployment. Without proper ONNX understanding, you're building inefficient models, struggling with framework compatibility, and missing the power of cross-platform deployment. This guide shows you how to wield ONNX with the precision of a machine learning engineer. @@ -1142,4 +1141,3 @@ ONNX model optimization provides the foundation for production machine learning --- -*This guide provides the complete machinery for mastering ONNX model optimization. The patterns scale from development to production, from simple conversions to complex deployment architectures.* diff --git a/docs/best-practices/ml-ai/prompting-llms.md b/docs/best-practices/ml-ai/prompting-llms.md index 39a7e7d..98ce2f1 100644 --- a/docs/best-practices/ml-ai/prompting-llms.md +++ b/docs/best-practices/ml-ai/prompting-llms.md @@ -1,6 +1,5 @@ # Best Practices: Prompting LLMs (with Agent Instructions) -**Objective**: Prompting isn't magic—it's controlled constraint. Your job: pin context, force structure, and require self-checks. The sections below include a Mermaid framing map, a table of prompt transformations, and a strict instruction block an agentic LLM can execute. ## Prompt Framing Map @@ -243,4 +242,3 @@ Miss any one and the model improvises; give all six and it builds. --- -*This guide provides the complete machinery for effective LLM prompting. Each technique is production-ready, copy-paste runnable, and designed for reliable, structured outputs with proper safety and verification.* diff --git a/docs/best-practices/ml-ai/r-data-exploration.md b/docs/best-practices/ml-ai/r-data-exploration.md index 1279462..61c1c64 100644 --- a/docs/best-practices/ml-ai/r-data-exploration.md +++ b/docs/best-practices/ml-ai/r-data-exploration.md @@ -1,6 +1,5 @@ # R Data Exploration: Tidyverse vs data.table (and Why You Should Prefer data.table) -**Objective**: Master R data exploration with a focus on production performance, scalability, and memory efficiency. When you need to handle large datasets, when you want maximum performance, when you're building production data pipelines—data.table becomes your weapon of choice. Tidyverse feels nice, but data.table keeps the lights on when the data gets ugly. While tidyverse excels at readability and teaching, data.table dominates when scale, speed, or reproducibility matter. This guide shows you how to wield data.table with the precision of a senior R engineer. @@ -871,4 +870,3 @@ R data exploration requires understanding both tidyverse and data.table ecosyste --- -*This guide provides the complete machinery for mastering R data exploration. The patterns scale from simple data manipulation to complex production pipelines, from basic analysis to advanced performance optimization.* diff --git a/docs/best-practices/ml-ai/vibe-to-agentic.md b/docs/best-practices/ml-ai/vibe-to-agentic.md index 845e424..6febcc1 100644 --- a/docs/best-practices/ml-ai/vibe-to-agentic.md +++ b/docs/best-practices/ml-ai/vibe-to-agentic.md @@ -1,6 +1,5 @@ # Vibe Coding → Heavy Prompts → Agentic Execution (2025 Edition) -**Objective**: Master the art of transforming creative "vibe coding" into structured, executable agentic workflows. Start with mood and intent; end with automated delivery. You are distilling chaos into contracts, and contracts into machines. ## The Pipeline (Mental Model) @@ -286,4 +285,3 @@ Return original structure with redactions. If none found, return unchanged. --- -*This guide provides the complete machinery for transforming creative vibes into structured, executable agentic workflows. Each pattern includes concrete prompts, schemas, and real-world implementation strategies for enterprise deployment.* diff --git a/docs/best-practices/operations-monitoring/blast-radius-risk-modeling.md b/docs/best-practices/operations-monitoring/blast-radius-risk-modeling.md index d5762d0..4387160 100644 --- a/docs/best-practices/operations-monitoring/blast-radius-risk-modeling.md +++ b/docs/best-practices/operations-monitoring/blast-radius-risk-modeling.md @@ -1,6 +1,5 @@ # Operational Risk Modeling, Blast Radius Reduction & Failure Domain Architecture: Best Practices -**Objective**: Establish comprehensive risk modeling frameworks that identify failure domains, model blast radius, and design containment strategies across clusters, databases, data pipelines, and ML systems. When you need to reduce risk, when you want to contain failures, when you need failure domain architecture—this guide provides the complete framework. ## Introduction diff --git a/docs/best-practices/operations-monitoring/chaos-engineering-governance.md b/docs/best-practices/operations-monitoring/chaos-engineering-governance.md index afbd59d..c1918a3 100644 --- a/docs/best-practices/operations-monitoring/chaos-engineering-governance.md +++ b/docs/best-practices/operations-monitoring/chaos-engineering-governance.md @@ -1,6 +1,5 @@ # Chaos Engineering, Fault Injection, and Reliability Validation Across Distributed Systems: Best Practices -**Objective**: Establish safe, systematic chaos engineering practices for validating system resilience across distributed systems, databases, microservices, and data pipelines. When you need to test failure scenarios, when you want to validate SLOs under stress, when you need reproducible fault injection—this guide provides the complete framework. ## Introduction diff --git a/docs/best-practices/operations-monitoring/configuration-drift-detection-prevention.md b/docs/best-practices/operations-monitoring/configuration-drift-detection-prevention.md index b7c9b29..7a3f446 100644 --- a/docs/best-practices/operations-monitoring/configuration-drift-detection-prevention.md +++ b/docs/best-practices/operations-monitoring/configuration-drift-detection-prevention.md @@ -1,6 +1,5 @@ # Cross-Environment Configuration Drift Detection & Drift-Proof Deployment Best Practices -**Objective**: Master production-grade configuration drift detection, prevention, and remediation across multi-environment distributed systems. When you need to ensure consistency across dev/stage/prod, prevent silent failures, and maintain reproducibility—this guide provides complete patterns and implementations. ## Introduction diff --git a/docs/best-practices/operations-monitoring/configuration-management.md b/docs/best-practices/operations-monitoring/configuration-management.md index 16bcc9a..bea5cbf 100644 --- a/docs/best-practices/operations-monitoring/configuration-management.md +++ b/docs/best-practices/operations-monitoring/configuration-management.md @@ -1,6 +1,5 @@ # Configuration Management, Secrets Lifecycle, and Multi-Environment Drift Control -**Objective**: Master production-grade configuration management across multi-environment distributed systems. When you need to prevent config drift, manage secrets securely, validate configurations, and maintain consistency across dev/stage/prod—this guide provides complete patterns and implementations. ## Introduction diff --git a/docs/best-practices/operations-monitoring/environment-promotion-drift-governance.md b/docs/best-practices/operations-monitoring/environment-promotion-drift-governance.md index 1f958ae..003d09d 100644 --- a/docs/best-practices/operations-monitoring/environment-promotion-drift-governance.md +++ b/docs/best-practices/operations-monitoring/environment-promotion-drift-governance.md @@ -1,6 +1,5 @@ # Cross-Environment Configuration Drift Prevention, Promotion Workflows & Release Channels: Best Practices -**Objective**: Establish comprehensive environment promotion governance that controls configuration flow from dev → staging → prod → isolated environments, prevents drift, and ensures immutable, versioned deployments. When you need controlled promotion, when you want drift prevention, when you need release channels—this guide provides the complete framework. ## Introduction diff --git a/docs/best-practices/operations-monitoring/grafana-prometheus-loki-observability.md b/docs/best-practices/operations-monitoring/grafana-prometheus-loki-observability.md index 80a5f74..8d0b8d4 100644 --- a/docs/best-practices/operations-monitoring/grafana-prometheus-loki-observability.md +++ b/docs/best-practices/operations-monitoring/grafana-prometheus-loki-observability.md @@ -1,6 +1,5 @@ # Best Practices for Monitoring & Observability with Grafana, Prometheus, Loki, Node Exporter, and Structured Logging -**Objective**: Master production-grade observability with Grafana, Prometheus, Loki, and Node Exporter. When you need comprehensive monitoring, structured logging, alerting, and dashboards at scale—these best practices become your foundation. ## Introduction @@ -2066,5 +2065,4 @@ This observability stack provides the foundation for understanding your systems --- -*This guide provides the complete machinery for production-grade observability. The patterns scale from single instances to enterprise deployments, from basic metrics to full distributed tracing.* diff --git a/docs/best-practices/operations-monitoring/grafana.md b/docs/best-practices/operations-monitoring/grafana.md index 703a76b..4b70011 100644 --- a/docs/best-practices/operations-monitoring/grafana.md +++ b/docs/best-practices/operations-monitoring/grafana.md @@ -1,6 +1,5 @@ # Grafana Best Practices -**Objective**: Master production-grade Grafana deployment, configuration, and operations for enterprise observability. When you need to build reliable monitoring dashboards, when you want to scale observability across teams, when you're responsible for keeping the lights on at 3AM—Grafana best practices become your weapon of choice. Grafana is not just a pretty dashboard tool. Treat it as production-critical software: upgrade it, provision it, secure it, observe it. @@ -1115,4 +1114,3 @@ Grafana requires understanding distributed systems, observability patterns, and --- -*This guide provides the complete machinery for production Grafana operations. The patterns scale from simple dashboards to complex enterprise monitoring, from basic alerting to advanced observability platforms.* diff --git a/docs/best-practices/operations-monitoring/index.md b/docs/best-practices/operations-monitoring/index.md index 9376b9e..5b0f606 100644 --- a/docs/best-practices/operations-monitoring/index.md +++ b/docs/best-practices/operations-monitoring/index.md @@ -1,6 +1,5 @@ # Operations & Monitoring Best Practices -**Objective**: Master senior-level operations and monitoring patterns for production systems. When you need to build robust, scalable operational systems, when you want to follow proven methodologies, when you need enterprise-grade patterns—these best practices become your weapon of choice. ## Performance & Reliability @@ -32,4 +31,3 @@ --- -*These best practices provide the complete machinery for building production-ready operational systems. Each guide includes architectural patterns, configuration examples, and real-world implementation strategies for enterprise deployment.* diff --git a/docs/best-practices/operations-monitoring/logging-observability.md b/docs/best-practices/operations-monitoring/logging-observability.md index bb76e17..5dda308 100644 --- a/docs/best-practices/operations-monitoring/logging-observability.md +++ b/docs/best-practices/operations-monitoring/logging-observability.md @@ -1,6 +1,5 @@ # Structured Logging & Observability Best Practices (2025 Edition) -**Objective**: Master production-grade logging and observability for modern data and DevOps environments. When you need to debug complex distributed systems, when you want to trace data flows across services, when you're responsible for keeping systems alive at 3AM—structured logging becomes your weapon of choice. Logs aren't for printing—they are lifelines. Make them structured, contextual, and queryable, or drown in noise. @@ -819,4 +818,3 @@ Structured logging requires understanding distributed systems, observability pat --- -*This guide provides the complete machinery for structured logging and observability. The patterns scale from simple application logging to complex distributed system monitoring, from basic log aggregation to advanced observability platforms.* diff --git a/docs/best-practices/operations-monitoring/observability-driven-development.md b/docs/best-practices/operations-monitoring/observability-driven-development.md index ceac368..a826b98 100644 --- a/docs/best-practices/operations-monitoring/observability-driven-development.md +++ b/docs/best-practices/operations-monitoring/observability-driven-development.md @@ -1,6 +1,5 @@ # Observability-Driven Development (ODD), Telemetry-First Coding Practices, and Preemptive Debugging Architecture: Best Practices -**Objective**: Establish comprehensive observability-driven development practices that embed telemetry, logging, tracing, and metrics as first-class design inputs from day one. When you need observability-first design, when you want preemptive debugging, when you need telemetry standards—this guide provides the complete framework. ## Introduction diff --git a/docs/best-practices/operations-monitoring/operational-resilience-and-incident-response.md b/docs/best-practices/operations-monitoring/operational-resilience-and-incident-response.md index d5af3d8..0aec479 100644 --- a/docs/best-practices/operations-monitoring/operational-resilience-and-incident-response.md +++ b/docs/best-practices/operations-monitoring/operational-resilience-and-incident-response.md @@ -1,6 +1,5 @@ # Operational Resilience and Incident Response: Best Practices for Distributed Systems -**Objective**: Master production-grade operational resilience across RKE2, Postgres, Prefect, Redis, ML pipelines, and air-gapped clusters. When you need to survive incidents, maintain uptime, and recover gracefully—this guide provides complete operational playbooks and incident response frameworks. ## Introduction diff --git a/docs/best-practices/operations-monitoring/release-management-and-progressive-delivery.md b/docs/best-practices/operations-monitoring/release-management-and-progressive-delivery.md index be92ff4..1370eed 100644 --- a/docs/best-practices/operations-monitoring/release-management-and-progressive-delivery.md +++ b/docs/best-practices/operations-monitoring/release-management-and-progressive-delivery.md @@ -7,7 +7,6 @@ tags: # Release Management, Change Governance, and Progressive Delivery: Best Practices for Distributed Systems -**Objective**: Master production-grade release management and progressive delivery for distributed systems. When you need to safely deploy changes across applications, databases, data pipelines, and ML systems—this guide provides complete patterns and implementations. ## Introduction @@ -101,7 +100,7 @@ graph TB - Ticket creation with labels **2. Design & ADR**: -- Architecture Decision Record (see [ADR Guide](../architecture-design/adr-decision-governance.md)) +- Architecture Decision Record (see [Documentation](../architecture-design/documentation.md)) - Design review - Risk classification @@ -1875,7 +1874,6 @@ kubectl apply -f manifests/k8s/ -n prod ## See Also -- **[ADR and Technical Decision Governance](../architecture-design/adr-decision-governance.md)** - Decision recording - **[Configuration Management](configuration-management.md)** - Config governance - **[System Resilience](../operations-monitoring/system-resilience-and-concurrency.md)** - Resilience patterns - **[Testing Best Practices](../operations-monitoring/testing-best-practices.md)** - Testing strategies diff --git a/docs/best-practices/operations-monitoring/secrets-config.md b/docs/best-practices/operations-monitoring/secrets-config.md index 467f232..6a93d5b 100644 --- a/docs/best-practices/operations-monitoring/secrets-config.md +++ b/docs/best-practices/operations-monitoring/secrets-config.md @@ -1,6 +1,5 @@ # Secrets & Configuration Management Best Practices (2025 Edition) -**Objective**: Master production-grade secrets and configuration management for modern data engineering and DevOps environments. When you need to secure sensitive data across environments, when you want to prevent credential exposure, when you're responsible for keeping secrets safe—proper secrets management becomes your weapon of choice. Secrets are the sharpest knives in your system. If you don't sheath them properly, you'll cut yourself—or worse, everyone else. @@ -810,4 +809,3 @@ Secret management requires understanding security patterns, access control, and --- -*This guide provides the complete machinery for secret and configuration management. The patterns scale from simple environment variables to complex enterprise secret stores, from basic access control to advanced security operations.* diff --git a/docs/best-practices/operations-monitoring/system-resilience-and-concurrency.md b/docs/best-practices/operations-monitoring/system-resilience-and-concurrency.md index fb6d1be..c44fe1f 100644 --- a/docs/best-practices/operations-monitoring/system-resilience-and-concurrency.md +++ b/docs/best-practices/operations-monitoring/system-resilience-and-concurrency.md @@ -1,2390 +1,110 @@ -# System Resilience, Rate Limiting, Concurrency Control & Backpressure: Best Practices for Distributed Systems +# System resilience, concurrency, and backpressure -**Objective**: Master production-grade resilience patterns for distributed systems. When you need to prevent cascading failures, control concurrency, implement rate limiting, handle backpressure, and maintain SLOs under load—this guide provides complete patterns and implementations. +Distributed systems fail when work enters faster than you can finish it. Connection pools exhaust, queues grow without bound, and one slow dependency stalls everything upstream. This page is about **bounding concurrency** and **signaling overload** before you run out of memory or threads. -## Introduction +It is not a catalog of every resilience pattern. It focuses on what I reach for first: limits, backpressure, and honest degradation. -Distributed systems face constant pressure: traffic spikes, resource exhaustion, network failures, and cascading errors. Without proper resilience patterns, systems fail catastrophically. This guide provides a complete framework for building resilient, high-performance distributed systems. +## When to care -**What This Guide Covers**: -- Concurrency control (threads, async, worker pools) -- Rate limiting (token bucket, leaky bucket, sliding windows) -- Backpressure mechanisms -- Load shedding and graceful degradation -- Circuit breakers and bulkheads -- SLO-driven design and error budgets -- Autoscaling strategies -- GIS and ML-specific resilience patterns +- **Postgres-heavy services**: pool size is a hard ceiling. Slow queries hold connections; retries multiply load. +- **Async Python / Go / Rust services**: goroutines or tasks are cheap until they are not — memory and file descriptors still cap you. +- **GIS and batch jobs**: large geometries and cold caches create latency spikes that look like “random” timeouts unless you limit fan-out. +- **ML inference**: GPU memory and batch queues need explicit bounds; “just add workers” often makes p95 worse. -**Prerequisites**: -- Understanding of distributed systems, microservices, and concurrency -- Familiarity with Python async, Go concurrency, Rust async -- Experience with Kubernetes, databases, and message queues +If the workload is a once-a-day cron, you probably do not need a circuit breaker. You need a timeout and an alert when the job overruns. -## Purpose & Importance +## Concurrency: pick a boundary and enforce it -### Why Resilience Patterns Are Essential +**Threads / processes (sync code)** +Use a fixed worker pool size derived from CPU and I/O, not `number_of_cpus * 100`. For DB-bound work, size against **pool connections**, not cores. -**Postgres-Heavy Systems**: -- Connection pool exhaustion causes cascading failures -- Slow queries block all requests -- Replication lag causes read inconsistencies -- Without rate limiting, databases are overwhelmed +**Async (Python asyncio, etc.)** +Semaphores limit in-flight requests. Without them, every accepted socket spawns unbounded task creation under load. -**GIS Computation Pipelines**: -- Spatial queries are CPU-intensive and slow -- Tile generation can saturate compute resources -- Isochrone calculations are expensive -- Without backpressure, GIS services become unresponsive +**Go** +Worker pools with buffered channels; respect `context` cancellation on shutdown. -**ML/ONNX Inference Flows**: -- Model loading is expensive (cold starts) -- GPU saturation causes queue buildup -- Feature extraction can be slow -- Without load shedding, inference latency degrades +**Rust** +`tokio::sync::Semaphore` or dedicated worker tasks; avoid unbounded `spawn` in handlers. -**Real-Time Dashboards**: -- Grafana queries can overwhelm databases -- NiceGUI WebSocket connections consume resources -- Real-time updates create thundering herd -- Without rate limiting, dashboards become unusable - -**ETL & Streaming Workloads**: -- Large datasets overwhelm memory -- Streaming backpressure causes data loss -- Pipeline failures cascade downstream -- Without circuit breakers, entire pipelines fail - -**Microservice Architectures**: -- Service failures cascade across dependencies -- Network partitions cause timeouts -- Retry storms overwhelm services -- Without isolation, one service failure takes down all - -**Air-Gapped Clusters**: -- Limited compute resources -- No external fallbacks -- Resource contention is severe -- Without load shedding, systems become unusable - -### Failure Cascade Without Resilience - -```mermaid -graph TD - Start["Traffic Spike"] --> API["API Service
(No Rate Limit)"] - API -->|"Unlimited Requests"| PG["Postgres
(Connection Exhaustion)"] - API -->|"Unlimited Requests"| Redis["Redis
(Memory Exhaustion)"] - API -->|"Unlimited Requests"| ML["ML Service
(GPU Saturation)"] - - PG -->|"Slow Queries"| API2["API Service
(Request Timeout)"] - Redis -->|"Evictions"| API3["API Service
(Cache Misses)"] - ML -->|"Queue Buildup"| API4["API Service
(Latency Spike)"] - - API2 -->|"Cascading Failures"| Down["System Down"] - API3 -->|"Cascading Failures"| Down - API4 -->|"Cascading Failures"| Down - - style Start fill:#ffcccc - style Down fill:#ff0000 -``` - -### Failure Cascade With Resilience - -```mermaid -graph TD - Start["Traffic Spike"] --> RL["Rate Limiter
(Token Bucket)"] - RL -->|"Throttled Requests"| API["API Service
(Controlled Load)"] - - API -->|"Pooled Connections"| PG["Postgres
(Connection Pool)"] - API -->|"Cached Responses"| Redis["Redis
(TTL + Eviction)"] - API -->|"Queued Requests"| ML["ML Service
(Worker Pool)"] - - PG -->|"Slow Query"| CB["Circuit Breaker
(Opens)"] - CB -->|"Fallback"| Cache["Cache Response"] - - ML -->|"Queue Full"| BP["Backpressure
(Reject New)"] - BP -->|"Load Shed"| LS["Load Shedder
(Drop Low Priority)"] - - Cache -->|"Graceful Degradation"| Stable["System Stable"] - LS -->|"Graceful Degradation"| Stable - - style Start fill:#ffcccc - style Stable fill:#ccffcc -``` - -## Core Concepts - -### Concurrency Control - -#### Thread-Level Concurrency - -**Python Threading**: - -```python -# concurrency/thread_pool.py -from concurrent.futures import ThreadPoolExecutor -import threading - -class ThreadPoolController: - def __init__(self, max_workers: int = 10): - self.max_workers = max_workers - self.executor = ThreadPoolExecutor(max_workers=max_workers) - self.semaphore = threading.Semaphore(max_workers) - - def submit(self, func, *args, **kwargs): - """Submit task with concurrency control""" - with self.semaphore: - return self.executor.submit(func, *args, **kwargs) - - def shutdown(self, wait=True): - """Shutdown thread pool""" - self.executor.shutdown(wait=wait) - -# Usage -pool = ThreadPoolController(max_workers=10) -future = pool.submit(expensive_operation, arg1, arg2) -result = future.result() -``` - -**Go Goroutine Pool**: - -```go -// concurrency/worker_pool.go -package concurrency - -import ( - "context" - "sync" -) - -type WorkerPool struct { - workers int - jobQueue chan func() - wg sync.WaitGroup -} - -func NewWorkerPool(workers int, queueSize int) *WorkerPool { - return &WorkerPool{ - workers: workers, - jobQueue: make(chan func(), queueSize), - } -} - -func (wp *WorkerPool) Start(ctx context.Context) { - for i := 0; i < wp.workers; i++ { - wp.wg.Add(1) - go func() { - defer wp.wg.Done() - for { - select { - case job := <-wp.jobQueue: - job() - case <-ctx.Done(): - return - } - } - }() - } -} - -func (wp *WorkerPool) Submit(job func()) error { - select { - case wp.jobQueue <- job: - return nil - default: - return ErrQueueFull - } -} - -func (wp *WorkerPool) Stop() { - close(wp.jobQueue) - wp.wg.Wait() -} -``` - -**Rust Tokio Concurrency**: - -```rust -// concurrency/tokio_pool.rs -use tokio::sync::Semaphore; -use std::sync::Arc; - -pub struct ConcurrencyLimiter { - semaphore: Arc, -} - -impl ConcurrencyLimiter { - pub fn new(max_concurrent: usize) -> Self { - Self { - semaphore: Arc::new(Semaphore::new(max_concurrent)), - } - } - - pub async fn execute(&self, f: F) -> T - where - F: Future, - { - let _permit = self.semaphore.acquire().await.unwrap(); - f.await - } -} - -// Usage -let limiter = ConcurrencyLimiter::new(10); -let result = limiter.execute(async { - expensive_operation().await -}).await; -``` - -#### Async Scheduling - -**Python asyncio**: - -```python -# concurrency/async_limiter.py -import asyncio -from asyncio import Semaphore - -class AsyncLimiter: - def __init__(self, max_concurrent: int = 10): - self.semaphore = Semaphore(max_concurrent) - - async def execute(self, coro): - """Execute coroutine with concurrency limit""" - async with self.semaphore: - return await coro - -# Usage -limiter = AsyncLimiter(max_concurrent=10) -results = await asyncio.gather(*[ - limiter.execute(process_item(item)) - for item in items -]) -``` - -#### Worker Pools - -**Python Worker Pool**: - -```python -# concurrency/worker_pool.py -import asyncio -from asyncio import Queue - -class AsyncWorkerPool: - def __init__(self, num_workers: int, queue_size: int = 100): - self.num_workers = num_workers - self.queue = Queue(maxsize=queue_size) - self.workers = [] - - async def start(self): - """Start worker pool""" - self.workers = [ - asyncio.create_task(self._worker()) - for _ in range(self.num_workers) - ] - - async def _worker(self): - """Worker coroutine""" - while True: - item = await self.queue.get() - if item is None: # Shutdown signal - break - await self.process_item(item) - self.queue.task_done() - - async def submit(self, item): - """Submit item to queue""" - await self.queue.put(item) - - async def shutdown(self): - """Shutdown worker pool""" - for _ in range(self.num_workers): - await self.queue.put(None) - await asyncio.gather(*self.workers) -``` - -#### CPU-Bound vs IO-Bound Decisions - -**Decision Matrix**: - -| Workload Type | Concurrency Model | Example | -|---------------|------------------|---------| -| **CPU-bound** | Thread pool (limited by CPU cores) | Image processing, ML inference | -| **IO-bound** | Async/await (many concurrent) | Database queries, API calls | -| **Mixed** | Hybrid (thread pool + async) | ETL with CPU transforms + DB I/O | - -**Implementation**: - -```python -# concurrency/cpu_vs_io.py -import asyncio -from concurrent.futures import ThreadPoolExecutor -import aiohttp - -class HybridExecutor: - def __init__(self, cpu_workers: int = 4, io_concurrent: int = 100): - self.cpu_pool = ThreadPoolExecutor(max_workers=cpu_workers) - self.io_semaphore = asyncio.Semaphore(io_concurrent) - - async def cpu_bound_task(self, func, *args): - """Execute CPU-bound task in thread pool""" - loop = asyncio.get_event_loop() - return await loop.run_in_executor(self.cpu_pool, func, *args) - - async def io_bound_task(self, coro): - """Execute IO-bound task with concurrency limit""" - async with self.io_semaphore: - return await coro - -# Usage -executor = HybridExecutor() - -# CPU-bound: ML inference -result = await executor.cpu_bound_task(model.predict, features) - -# IO-bound: API call -response = await executor.io_bound_task(session.get(url)) -``` - -#### Controlling Parallel Geospatial Computations - -**Pattern**: Limit concurrent spatial operations. - -```python -# concurrency/geospatial_limiter.py -import asyncio -from shapely.geometry import LineString - -class GeospatialLimiter: - def __init__(self, max_concurrent: int = 4): - self.semaphore = asyncio.Semaphore(max_concurrent) - - async def compute_intersection(self, geom1, geom2): - """Compute spatial intersection with concurrency limit""" - async with self.semaphore: - # CPU-bound operation in thread pool - loop = asyncio.get_event_loop() - return await loop.run_in_executor( - None, - lambda: geom1.intersects(geom2) - ) - -# Usage -limiter = GeospatialLimiter(max_concurrent=4) -results = await asyncio.gather(*[ - limiter.compute_intersection(road, zone) - for road, zone in road_zone_pairs -]) -``` - -#### Concurrency in Go, Python Async, Rust Async - -**Comparison Table**: - -| Language | Concurrency Model | Best For | -|----------|------------------|----------| -| **Go** | Goroutines + channels | High-throughput, simple concurrency | -| **Python async** | asyncio + coroutines | IO-bound, event-driven | -| **Rust async** | Tokio + futures | High-performance, memory-safe | - -**Go Example**: - -```go -// concurrency/go_concurrency.go -func processConcurrently(items []Item) []Result { - results := make([]Result, len(items)) - sem := make(chan struct{}, 10) // Max 10 concurrent - var wg sync.WaitGroup - - for i, item := range items { - wg.Add(1) - go func(idx int, it Item) { - defer wg.Done() - sem <- struct{}{} // Acquire - defer func() { <-sem }() // Release - - results[idx] = processItem(it) - }(i, item) - } - - wg.Wait() - return results -} -``` - -**Python Async Example**: - -```python -# concurrency/python_async.py -async def process_concurrently(items: list) -> list: - semaphore = asyncio.Semaphore(10) # Max 10 concurrent - - async def process_item(item): - async with semaphore: - return await process_item_async(item) - - return await asyncio.gather(*[ - process_item(item) for item in items - ]) -``` - -**Rust Async Example**: - -```rust -// concurrency/rust_async.rs -use tokio::sync::Semaphore; - -async fn process_concurrently(items: Vec) -> Vec { - let semaphore = Arc::new(Semaphore::new(10)); // Max 10 concurrent - - let tasks: Vec<_> = items.into_iter().map(|item| { - let sem = semaphore.clone(); - tokio::spawn(async move { - let _permit = sem.acquire().await.unwrap(); - process_item(item).await - }) - }).collect(); - - futures::future::join_all(tasks).await - .into_iter() - .map(|r| r.unwrap()) - .collect() -} -``` - -#### Controlling ONNX Inference Concurrency - -**Pattern**: Limit concurrent model inference. - -```python -# concurrency/onnx_limiter.py -import onnxruntime as ort -import asyncio - -class ONNXInferenceLimiter: - def __init__(self, model_path: str, max_concurrent: int = 2): - self.session = ort.InferenceSession(model_path) - self.semaphore = asyncio.Semaphore(max_concurrent) - - async def infer(self, input_data: np.ndarray): - """Run inference with concurrency limit""" - async with self.semaphore: - # Run inference (CPU-bound, but async for concurrency control) - loop = asyncio.get_event_loop() - return await loop.run_in_executor( - None, - lambda: self.session.run(None, {"input": input_data}) - ) -``` - -### Rate Limiting Patterns - -#### Token Bucket - -**Algorithm**: Tokens added at fixed rate, requests consume tokens. - -**Python Implementation**: - -```python -# rate_limiting/token_bucket.py -import time -import asyncio -from collections import deque - -class TokenBucket: - def __init__(self, capacity: int, refill_rate: float): - """ - capacity: Maximum tokens - refill_rate: Tokens per second - """ - self.capacity = capacity - self.refill_rate = refill_rate - self.tokens = capacity - self.last_refill = time.time() - self.lock = asyncio.Lock() - - async def acquire(self, tokens: int = 1) -> bool: - """Try to acquire tokens""" - async with self.lock: - self._refill() - - if self.tokens >= tokens: - self.tokens -= tokens - return True - return False - - def _refill(self): - """Refill tokens based on elapsed time""" - now = time.time() - elapsed = now - self.last_refill - self.tokens = min( - self.capacity, - self.tokens + elapsed * self.refill_rate - ) - self.last_refill = now - -# Usage -bucket = TokenBucket(capacity=100, refill_rate=10.0) # 10 tokens/sec - -async def make_request(): - if await bucket.acquire(): - # Make request - pass - else: - # Rate limited - raise RateLimitError() -``` - -**Redis-Based Token Bucket**: - -```python -# rate_limiting/redis_token_bucket.py -import redis -import time - -class RedisTokenBucket: - def __init__(self, redis_client: redis.Redis, key: str, capacity: int, refill_rate: float): - self.redis = redis_client - self.key = key - self.capacity = capacity - self.refill_rate = refill_rate - - def acquire(self, tokens: int = 1) -> bool: - """Acquire tokens using Redis Lua script""" - lua_script = """ - local key = KEYS[1] - local capacity = tonumber(ARGV[1]) - local refill_rate = tonumber(ARGV[2]) - local tokens_requested = tonumber(ARGV[3]) - local now = tonumber(ARGV[4]) - - local bucket = redis.call('HMGET', key, 'tokens', 'last_refill') - local current_tokens = tonumber(bucket[1]) or capacity - local last_refill = tonumber(bucket[2]) or now - - -- Refill tokens - local elapsed = now - last_refill - current_tokens = math.min(capacity, current_tokens + elapsed * refill_rate) - - -- Check if enough tokens - if current_tokens >= tokens_requested then - current_tokens = current_tokens - tokens_requested - redis.call('HMSET', key, 'tokens', current_tokens, 'last_refill', now) - redis.call('EXPIRE', key, 3600) - return 1 - else - redis.call('HMSET', key, 'tokens', current_tokens, 'last_refill', now) - redis.call('EXPIRE', key, 3600) - return 0 - end - """ - - result = self.redis.eval( - lua_script, - 1, - self.key, - self.capacity, - self.refill_rate, - tokens, - int(time.time()) - ) - - return bool(result) -``` - -#### Leaky Bucket - -**Algorithm**: Requests added to bucket, processed at fixed rate. - -**Python Implementation**: - -```python -# rate_limiting/leaky_bucket.py -import asyncio -import time -from collections import deque - -class LeakyBucket: - def __init__(self, capacity: int, leak_rate: float): - """ - capacity: Maximum queue size - leak_rate: Requests per second processed - """ - self.capacity = capacity - self.leak_rate = leak_rate - self.queue = deque() - self.last_leak = time.time() - self.lock = asyncio.Lock() - - async def add(self, item) -> bool: - """Add item to bucket""" - async with self.lock: - if len(self.queue) >= self.capacity: - return False # Bucket full - - self.queue.append((item, time.time())) - return True - - async def process(self): - """Process items from bucket at leak rate""" - while True: - async with self.lock: - self._leak() - - await asyncio.sleep(1.0 / self.leak_rate) - - def _leak(self): - """Remove items based on leak rate""" - now = time.time() - elapsed = now - self.last_leak - items_to_process = int(elapsed * self.leak_rate) - - for _ in range(min(items_to_process, len(self.queue))): - if self.queue: - item, _ = self.queue.popleft() - # Process item - asyncio.create_task(process_item(item)) - - self.last_leak = now -``` - -#### Fixed Window - -**Algorithm**: Count requests in fixed time window. - -**Python Implementation**: - -```python -# rate_limiting/fixed_window.py -import time -from collections import defaultdict - -class FixedWindowLimiter: - def __init__(self, max_requests: int, window_seconds: int): - self.max_requests = max_requests - self.window_seconds = window_seconds - self.windows = defaultdict(int) - self.window_start = defaultdict(int) - - def is_allowed(self, key: str) -> bool: - """Check if request is allowed""" - now = int(time.time()) - window = now // self.window_seconds - - # Reset if new window - if self.window_start[key] != window: - self.windows[key] = 0 - self.window_start[key] = window - - # Check limit - if self.windows[key] < self.max_requests: - self.windows[key] += 1 - return True - - return False -``` - -**Redis Fixed Window**: - -```python -# rate_limiting/redis_fixed_window.py -import redis -import time - -class RedisFixedWindowLimiter: - def __init__(self, redis_client: redis.Redis, max_requests: int, window_seconds: int): - self.redis = redis_client - self.max_requests = max_requests - self.window_seconds = window_seconds - - def is_allowed(self, key: str) -> bool: - """Check if request is allowed using Redis""" - window = int(time.time()) // self.window_seconds - redis_key = f"rate_limit:{key}:{window}" - - current = self.redis.incr(redis_key) - self.redis.expire(redis_key, self.window_seconds) - - return current <= self.max_requests -``` - -#### Sliding Log / Rolling Window - -**Algorithm**: Track individual request timestamps, count in sliding window. - -**Python Implementation**: - -```python -# rate_limiting/sliding_log.py -import time -from collections import deque - -class SlidingLogLimiter: - def __init__(self, max_requests: int, window_seconds: int): - self.max_requests = max_requests - self.window_seconds = window_seconds - self.logs = defaultdict(deque) - - def is_allowed(self, key: str) -> bool: - """Check if request is allowed""" - now = time.time() - window_start = now - self.window_seconds - - # Remove old entries - log = self.logs[key] - while log and log[0] < window_start: - log.popleft() - - # Check limit - if len(log) < self.max_requests: - log.append(now) - return True - - return False -``` - -**Redis Sliding Log**: - -```python -# rate_limiting/redis_sliding_log.py -import redis -import time - -class RedisSlidingLogLimiter: - def __init__(self, redis_client: redis.Redis, max_requests: int, window_seconds: int): - self.redis = redis_client - self.max_requests = max_requests - self.window_seconds = window_seconds - - def is_allowed(self, key: str) -> bool: - """Check if request is allowed using Redis sorted set""" - now = time.time() - window_start = now - self.window_seconds - redis_key = f"rate_limit:sliding:{key}" - - # Remove old entries - self.redis.zremrangebyscore(redis_key, 0, window_start) - - # Count current entries - current = self.redis.zcard(redis_key) - - if current < self.max_requests: - # Add current request - self.redis.zadd(redis_key, {str(now): now}) - self.redis.expire(redis_key, self.window_seconds) - return True - - return False -``` - -#### NGINX-Based Rate Limits - -**NGINX Configuration**: - -```nginx -# nginx/rate_limiting.conf -http { - # Define rate limit zones - limit_req_zone $binary_remote_addr zone=api_limit:10m rate=10r/s; - limit_req_zone $http_authorization zone=user_limit:10m rate=100r/s; - - # Per-IP rate limiting - limit_conn_zone $binary_remote_addr zone=conn_limit:10m; - - server { - listen 80; - server_name api.example.com; - - # Apply rate limits - limit_req zone=api_limit burst=20 nodelay; - limit_conn conn_limit 10; - - location /api/ { - proxy_pass http://backend; - - # Per-user rate limiting (if auth header present) - limit_req zone=user_limit burst=50 nodelay; - } - - # Stricter limits for expensive endpoints - location /api/ml/infer { - limit_req zone=api_limit burst=5 nodelay; - proxy_pass http://ml-backend; - } - } -} -``` - -#### Per-User, Per-IP, Per-Org, Per-Service Policies - -**Multi-Level Rate Limiting**: - -```python -# rate_limiting/multi_level.py -class MultiLevelRateLimiter: - def __init__(self, redis_client: redis.Redis): - self.redis = redis_client - self.limiters = { - "ip": RedisFixedWindowLimiter(redis_client, 100, 60), # 100/min per IP - "user": RedisFixedWindowLimiter(redis_client, 1000, 60), # 1000/min per user - "org": RedisFixedWindowLimiter(redis_client, 10000, 60), # 10000/min per org - "service": RedisFixedWindowLimiter(redis_client, 100000, 60) # 100000/min per service - } - - def is_allowed(self, ip: str, user_id: str, org_id: str, service: str) -> bool: - """Check all rate limits""" - checks = [ - self.limiters["ip"].is_allowed(f"ip:{ip}"), - self.limiters["user"].is_allowed(f"user:{user_id}"), - self.limiters["org"].is_allowed(f"org:{org_id}"), - self.limiters["service"].is_allowed(f"service:{service}") - ] - - return all(checks) -``` - -#### Rate Limiting Downstream Resources - -**Protecting Postgres**: - -```python -# rate_limiting/protect_postgres.py -class PostgresRateLimiter: - def __init__(self, redis_client: redis.Redis): - self.redis = redis_client - self.limiter = RedisTokenBucket( - redis_client, - "postgres:rate_limit", - capacity=1000, # Max 1000 queries - refill_rate=100.0 # 100 queries/sec - ) - - async def execute_query(self, query: str, params: tuple): - """Execute query with rate limiting""" - if not await self.limiter.acquire(): - raise RateLimitError("Postgres rate limit exceeded") - - return await db.execute(query, params) -``` - -### Backpressure Mechanisms - -#### Queue Size Controls - -**Pattern**: Reject requests when queue is full. - -```python -# backpressure/queue_control.py -import asyncio -from asyncio import Queue, QueueFull - -class BackpressureQueue: - def __init__(self, maxsize: int = 100): - self.queue = Queue(maxsize=maxsize) - self.maxsize = maxsize - - async def put(self, item, timeout: float = 0.1) -> bool: - """Put item with timeout""" - try: - await asyncio.wait_for(self.queue.put(item), timeout=timeout) - return True - except asyncio.TimeoutError: - return False # Queue full, backpressure - - async def get(self): - """Get item from queue""" - return await self.queue.get() -``` - -#### Async Task Backpressure - -**Pattern**: Limit concurrent async tasks. - -```python -# backpressure/async_backpressure.py -class AsyncBackpressure: - def __init__(self, max_concurrent: int = 10, max_queue: int = 100): - self.semaphore = asyncio.Semaphore(max_concurrent) - self.queue = asyncio.Queue(maxsize=max_queue) - - async def submit(self, coro) -> bool: - """Submit task with backpressure""" - try: - await asyncio.wait_for(self.queue.put(coro), timeout=0.1) - asyncio.create_task(self._process()) - return True - except asyncio.TimeoutError: - return False # Backpressure: queue full - - async def _process(self): - """Process tasks from queue""" - async with self.semaphore: - coro = await self.queue.get() - await coro -``` - -#### Redis Streams Backpressure - -**Pattern**: Monitor stream length, reject when too high. - -```python -# backpressure/redis_streams.py -class RedisStreamBackpressure: - def __init__(self, redis_client: redis.Redis, stream_name: str, max_length: int = 10000): - self.redis = redis_client - self.stream_name = stream_name - self.max_length = max_length - - async def add_message(self, fields: dict) -> bool: - """Add message with backpressure check""" - # Check stream length - length = self.redis.xlen(self.stream_name) - - if length >= self.max_length: - return False # Backpressure: stream too long - - # Add message - self.redis.xadd(self.stream_name, fields, maxlen=self.max_length) - return True -``` - -#### Prefect/Dask Backpressure - -**Prefect Task Concurrency Limits**: - -```python -# backpressure/prefect_backpressure.py -from prefect import task, flow -from prefect.concurrency import concurrency - -@task -@concurrency(limit=10, occupy=1) # Max 10 concurrent -def expensive_task(data: dict): - """Task with concurrency limit""" - return process_data(data) - -@flow -def etl_flow(): - """ETL flow with backpressure""" - items = load_items() - - # Process with concurrency limit (backpressure) - results = [] - for item in items: - result = expensive_task(item) # Will queue if limit reached - results.append(result) - - return results -``` - -**Dask Backpressure**: - -```python -# backpressure/dask_backpressure.py -from dask.distributed import Client -from dask import delayed - -def process_with_backpressure(client: Client, items: list, max_concurrent: int = 10): - """Process items with Dask backpressure""" - # Limit concurrent tasks - semaphore = asyncio.Semaphore(max_concurrent) - - @delayed - def process_item(item): - return expensive_operation(item) - - # Process with limit - tasks = [process_item(item) for item in items] - results = client.compute(tasks, resources={"worker": 1}) - - return results -``` - -#### Postgres Connection Pool Saturation - -**Pattern**: Reject connections when pool is full. - -```python -# backpressure/postgres_pool.py -import asyncpg -from asyncpg import Pool - -class PostgresPoolWithBackpressure: - def __init__(self, dsn: str, min_size: int = 5, max_size: int = 20): - self.pool: Pool = None - self.dsn = dsn - self.min_size = min_size - self.max_size = max_size - - async def connect(self): - """Create connection pool""" - self.pool = await asyncpg.create_pool( - self.dsn, - min_size=self.min_size, - max_size=self.max_size - ) - - async def execute(self, query: str, *args, timeout: float = 5.0): - """Execute query with timeout and backpressure""" - try: - async with self.pool.acquire(timeout=timeout) as conn: - return await conn.execute(query, *args) - except asyncio.TimeoutError: - raise BackpressureError("Connection pool exhausted") -``` - -**Postgres max_connections**: - -```sql --- postgresql.conf -max_connections = 100 -superuser_reserved_connections = 3 - --- Per-database connection limits -ALTER DATABASE mydb CONNECTION LIMIT 50; -``` - -#### GIS Tile Generation Throttling - -**Pattern**: Limit concurrent tile generation. - -```python -# backpressure/gis_tile_throttle.py -class TileGenerationThrottle: - def __init__(self, max_concurrent: int = 4): - self.semaphore = asyncio.Semaphore(max_concurrent) - - async def generate_tile(self, z: int, x: int, y: int): - """Generate tile with throttling""" - async with self.semaphore: - return await generate_tile_async(z, x, y) -``` - -#### Model Inference Backpressure - -**Pattern**: Queue inference requests, reject when queue full. - -```python -# backpressure/ml_inference.py -class MLInferenceBackpressure: - def __init__(self, model, max_queue: int = 100, max_concurrent: int = 2): - self.model = model - self.queue = asyncio.Queue(maxsize=max_queue) - self.semaphore = asyncio.Semaphore(max_concurrent) - - async def infer(self, input_data: np.ndarray, timeout: float = 5.0) -> np.ndarray: - """Run inference with backpressure""" - try: - # Try to add to queue - future = asyncio.Future() - await asyncio.wait_for(self.queue.put((input_data, future)), timeout=0.1) - - # Process inference - asyncio.create_task(self._process_inference()) - - # Wait for result - return await asyncio.wait_for(future, timeout=timeout) - except asyncio.TimeoutError: - raise BackpressureError("Inference queue full") - - async def _process_inference(self): - """Process inference requests""" - async with self.semaphore: - input_data, future = await self.queue.get() - result = self.model.predict(input_data) - future.set_result(result) -``` - -### Load Shedding - -#### Soft vs Hard Shedding - -**Soft Shedding**: Delay or degrade service. - -**Hard Shedding**: Reject requests immediately. - -```python -# load_shedding/shedding_strategy.py -class LoadShedder: - def __init__(self, soft_threshold: float = 0.8, hard_threshold: float = 0.95): - self.soft_threshold = soft_threshold - self.hard_threshold = hard_threshold - - def should_shed(self, load: float, priority: str = "normal") -> str: - """Determine shedding strategy""" - if load >= self.hard_threshold: - return "hard" # Reject immediately - - if load >= self.soft_threshold: - if priority == "low": - return "soft" # Delay or degrade - return "none" - - return "none" -``` - -#### Prioritized Workloads - -**Pattern**: Shed low-priority requests first. - -```python -# load_shedding/priority_shedding.py -from enum import Enum - -class Priority(Enum): - CRITICAL = 1 - HIGH = 2 - NORMAL = 3 - LOW = 4 - -class PriorityLoadShedder: - def __init__(self, max_load: float = 0.9): - self.max_load = max_load - - def should_process(self, priority: Priority, current_load: float) -> bool: - """Determine if request should be processed""" - if current_load < self.max_load: - return True - - # Shed based on priority - if priority == Priority.CRITICAL: - return True - elif priority == Priority.HIGH and current_load < 0.95: - return True - elif priority == Priority.NORMAL and current_load < 0.85: - return True - else: - return False # Shed low priority -``` - -#### Shedding Low-Priority Traffic - -**Implementation**: - -```python -# load_shedding/traffic_shedding.py -class TrafficShedder: - def __init__(self, redis_client: redis.Redis): - self.redis = redis_client - self.load_monitor = LoadMonitor() - - async def handle_request(self, request: Request) -> Response: - """Handle request with load shedding""" - current_load = await self.load_monitor.get_load() - priority = request.headers.get("X-Priority", "normal") - - if not self.should_process(priority, current_load): - return Response( - status_code=503, - body={"error": "Service overloaded, please retry later"} - ) - - return await process_request(request) -``` - -#### Shedding ML Inference When Models Are Cold - -**Pattern**: Reject requests when model is loading. - -```python -# load_shedding/ml_cold_start.py -class MLColdStartShedder: - def __init__(self, model_loader): - self.model_loader = model_loader - self.model_loaded = asyncio.Event() - self.loading = False - - async def load_model(self): - """Load model asynchronously""" - self.loading = True - try: - await self.model_loader.load() - self.model_loaded.set() - finally: - self.loading = False - - async def infer(self, input_data: np.ndarray) -> np.ndarray: - """Run inference, shed if model not loaded""" - if not self.model_loaded.is_set(): - if self.loading: - raise ServiceUnavailableError("Model loading, please retry") - else: - # Start loading - asyncio.create_task(self.load_model()) - raise ServiceUnavailableError("Model not loaded") - - return await self.model.infer(input_data) -``` - -#### Dropping Expensive Geospatial Queries - -**Pattern**: Reject expensive spatial queries under load. +Illustrative pattern (Python asyncio): ```python -# load_shedding/geospatial_shedding.py -class GeospatialLoadShedder: - def __init__(self, max_concurrent_queries: int = 10): - self.semaphore = asyncio.Semaphore(max_concurrent_queries) - self.queue_size = 0 - - async def execute_query(self, query: str, timeout: float = 30.0) -> dict: - """Execute geospatial query with load shedding""" - # Check if we can accept query - if self.queue_size > 50: # Threshold - raise LoadShedError("Too many geospatial queries in queue") - - try: - async with asyncio.timeout(timeout): - async with self.semaphore: - self.queue_size += 1 - try: - return await execute_spatial_query(query) - finally: - self.queue_size -= 1 - except asyncio.TimeoutError: - raise LoadShedError("Geospatial query timeout") -``` - -#### Returning Partial or Approximate Results - -**Pattern**: Return cached or approximate results under load. - -```python -# load_shedding/approximate_results.py -class ApproximateResultShedder: - def __init__(self, cache_client: redis.Redis): - self.cache = cache_client - - async def get_result(self, query: str, exact: bool = True) -> dict: - """Get result, return approximate if under load""" - current_load = await get_system_load() - - if current_load > 0.9 and not exact: - # Return cached approximate result - cached = self.cache.get(f"approx:{hash(query)}") - if cached: - return json.loads(cached) - - # Compute exact result - result = await compute_exact_result(query) - - # Cache for future approximate results - self.cache.setex( - f"approx:{hash(query)}", - 3600, - json.dumps(result) - ) - - return result -``` - -### Circuit Breakers - -#### Open/Half-Open/Closed States - -**State Machine**: - -```mermaid -stateDiagram-v2 - [*] --> Closed: Initial - Closed --> Open: Failure threshold exceeded - Open --> HalfOpen: Timeout elapsed - HalfOpen --> Closed: Success - HalfOpen --> Open: Failure -``` - -**Python Implementation**: - -```python -# circuit_breaker/circuit_breaker.py -import asyncio -import time -from enum import Enum - -class CircuitState(Enum): - CLOSED = "closed" - OPEN = "open" - HALF_OPEN = "half_open" - -class CircuitBreaker: - def __init__( - self, - failure_threshold: int = 5, - success_threshold: int = 2, - timeout: float = 60.0 - ): - self.failure_threshold = failure_threshold - self.success_threshold = success_threshold - self.timeout = timeout - - self.state = CircuitState.CLOSED - self.failure_count = 0 - self.success_count = 0 - self.last_failure_time = None - self.lock = asyncio.Lock() - - async def call(self, func, *args, **kwargs): - """Call function with circuit breaker""" - async with self.lock: - if self.state == CircuitState.OPEN: - if time.time() - self.last_failure_time > self.timeout: - self.state = CircuitState.HALF_OPEN - self.success_count = 0 - else: - raise CircuitBreakerOpenError("Circuit breaker is open") - - try: - result = await func(*args, **kwargs) - await self._on_success() - return result - except Exception as e: - await self._on_failure() - raise - - async def _on_success(self): - """Handle successful call""" - async with self.lock: - if self.state == CircuitState.HALF_OPEN: - self.success_count += 1 - if self.success_count >= self.success_threshold: - self.state = CircuitState.CLOSED - self.failure_count = 0 - elif self.state == CircuitState.CLOSED: - self.failure_count = 0 - - async def _on_failure(self): - """Handle failed call""" - async with self.lock: - self.failure_count += 1 - self.last_failure_time = time.time() - - if self.failure_count >= self.failure_threshold: - self.state = CircuitState.OPEN - elif self.state == CircuitState.HALF_OPEN: - self.state = CircuitState.OPEN -``` - -#### Per-Service Circuit Breaking - -**Multi-Service Circuit Breaker**: - -```python -# circuit_breaker/multi_service.py -class MultiServiceCircuitBreaker: - def __init__(self): - self.breakers = {} - - def get_breaker(self, service_name: str) -> CircuitBreaker: - """Get circuit breaker for service""" - if service_name not in self.breakers: - self.breakers[service_name] = CircuitBreaker() - return self.breakers[service_name] - - async def call_service(self, service_name: str, func, *args, **kwargs): - """Call service with circuit breaker""" - breaker = self.get_breaker(service_name) - return await breaker.call(func, *args, **kwargs) -``` - -#### DB Circuit Breaking - -**Postgres Circuit Breaker**: - -```python -# circuit_breaker/db_circuit_breaker.py -class PostgresCircuitBreaker: - def __init__(self, pool: asyncpg.Pool): - self.pool = pool - self.breaker = CircuitBreaker( - failure_threshold=5, - timeout=30.0 - ) - - async def execute(self, query: str, *args): - """Execute query with circuit breaker""" - async def _execute(): - async with self.pool.acquire() as conn: - return await conn.execute(query, *args) - - return await self.breaker.call(_execute) -``` - -#### ML Inference Circuit Breaking - -**ONNX Circuit Breaker**: - -```python -# circuit_breaker/ml_circuit_breaker.py -class MLInferenceCircuitBreaker: - def __init__(self, model_session: ort.InferenceSession): - self.session = model_session - self.breaker = CircuitBreaker( - failure_threshold=3, - timeout=60.0 - ) - - async def infer(self, input_data: np.ndarray) -> np.ndarray: - """Run inference with circuit breaker""" - async def _infer(): - loop = asyncio.get_event_loop() - return await loop.run_in_executor( - None, - lambda: self.session.run(None, {"input": input_data}) - ) - - return await self.breaker.call(_infer) -``` - -#### Redis Failure Fallback - -**Pattern**: Fallback to database when Redis fails. - -```python -# circuit_breaker/redis_fallback.py -class RedisWithFallback: - def __init__(self, redis_client: redis.Redis, db_client): - self.redis = redis_client - self.db = db_client - self.breaker = CircuitBreaker(failure_threshold=3, timeout=30.0) - - async def get(self, key: str): - """Get value with Redis fallback to DB""" - async def _get_from_redis(): - return await self.redis.get(key) - - try: - return await self.breaker.call(_get_from_redis) - except CircuitBreakerOpenError: - # Fallback to database - return await self.db.get(key) -``` - -#### NGINX Circuit Breaking via Upstream Health Checks - -**NGINX Configuration**: - -```nginx -# nginx/circuit_breaker.conf -upstream backend { - server backend1:8000 max_fails=3 fail_timeout=30s; - server backend2:8000 max_fails=3 fail_timeout=30s backup; - - # Health check - health_check interval=10s fails=3 passes=2; -} - -server { - location /api/ { - proxy_pass http://backend; - - # Circuit breaker behavior - proxy_next_upstream error timeout http_500 http_502 http_503; - proxy_next_upstream_tries 3; - proxy_next_upstream_timeout 10s; - } -} -``` - -### Bulkheads / Isolation - -#### Workload Isolation - -**Pattern**: Isolate workloads to prevent cascading failures. - -```python -# bulkheads/workload_isolation.py -class WorkloadIsolator: - def __init__(self): - self.pools = { - "user_facing": asyncio.Semaphore(10), - "etl": asyncio.Semaphore(5), - "ml": asyncio.Semaphore(2), - "gis": asyncio.Semaphore(4) - } - - async def execute(self, workload_type: str, coro): - """Execute workload in isolated pool""" - semaphore = self.pools.get(workload_type) - if not semaphore: - raise ValueError(f"Unknown workload type: {workload_type}") - - async with semaphore: - return await coro -``` - -#### Query Isolation in Postgres - -**Postgres Resource Groups**: - -```sql --- Create resource groups -CREATE RESOURCE GROUP user_queries WITH ( - cpu_rate_limit = 50, - memory_limit = 50 -); - -CREATE RESOURCE GROUP etl_queries WITH ( - cpu_rate_limit = 30, - memory_limit = 30 -); - --- Assign roles to resource groups -ALTER ROLE app_user SET resource_group = user_queries; -ALTER ROLE etl_user SET resource_group = etl_queries; -``` - -**Connection Pool Isolation**: +# ILLUSTRATIVE — tune limits from pool size and measured latency +MAX_IN_FLIGHT = 32 +sem = asyncio.Semaphore(MAX_IN_FLIGHT) -```python -# bulkheads/postgres_isolation.py -class IsolatedPostgresPools: - def __init__(self, dsn: str): - self.pools = { - "user_facing": None, - "etl": None, - "reporting": None - } - self.dsn = dsn - - async def connect(self): - """Create isolated connection pools""" - self.pools["user_facing"] = await asyncpg.create_pool( - self.dsn, - min_size=10, - max_size=20, - command_timeout=5.0 - ) - - self.pools["etl"] = await asyncpg.create_pool( - self.dsn, - min_size=5, - max_size=10, - command_timeout=300.0 - ) - - self.pools["reporting"] = await asyncpg.create_pool( - self.dsn, - min_size=2, - max_size=5, - command_timeout=60.0 - ) - - async def execute(self, pool_name: str, query: str, *args): - """Execute query in isolated pool""" - pool = self.pools.get(pool_name) - if not pool: - raise ValueError(f"Unknown pool: {pool_name}") - - async with pool.acquire() as conn: - return await conn.execute(query, *args) -``` - -#### Separate Redis Pools - -**Redis Connection Pool Isolation**: - -```python -# bulkheads/redis_isolation.py -class IsolatedRedisPools: - def __init__(self): - self.pools = { - "cache": redis.ConnectionPool(host="redis", db=0, max_connections=50), - "sessions": redis.ConnectionPool(host="redis", db=1, max_connections=20), - "streams": redis.ConnectionPool(host="redis", db=2, max_connections=10) - } - - def get_client(self, pool_name: str) -> redis.Redis: - """Get Redis client for isolated pool""" - pool = self.pools.get(pool_name) - if not pool: - raise ValueError(f"Unknown pool: {pool_name}") - - return redis.Redis(connection_pool=pool) -``` - -#### Separate ONNX Inference Worker Pools - -**ML Worker Pool Isolation**: - -```python -# bulkheads/ml_isolation.py -class IsolatedMLPools: - def __init__(self): - self.pools = { - "real_time": AsyncWorkerPool(num_workers=4, queue_size=100), - "batch": AsyncWorkerPool(num_workers=2, queue_size=1000), - "experimental": AsyncWorkerPool(num_workers=1, queue_size=50) - } - - async def infer(self, pool_name: str, input_data: np.ndarray) -> np.ndarray: - """Run inference in isolated pool""" - pool = self.pools.get(pool_name) - if not pool: - raise ValueError(f"Unknown pool: {pool_name}") - - return await pool.submit(lambda: model.infer(input_data)) -``` - -### Latency SLOs & Error Budgets - -#### Define Latency SLOs - -**SLO Definitions**: - -```yaml -# slos/latency_slos.yaml -slos: - api: - p50: 50ms - p95: 200ms - p99: 500ms - availability: 99.9% - - postgres: - p50: 10ms - p95: 100ms - p99: 500ms - availability: 99.95% - - ml_inference: - p50: 50ms - p95: 200ms - p99: 500ms - availability: 99.5% - - gis_isochrone: - p50: 1s - p95: 5s - p99: 10s - availability: 99.0% -``` - -**Prometheus SLO Tracking**: - -```python -# slos/slo_tracking.py -from prometheus_client import Histogram, Counter - -# Latency histograms -api_latency = Histogram( - 'api_request_duration_seconds', - 'API request latency', - buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0] -) - -# Error budget -error_budget = Counter( - 'slo_error_budget_consumed', - 'SLO error budget consumed', - ['service', 'slo_type'] -) - -def track_api_latency(duration: float): - """Track API latency""" - api_latency.observe(duration) - - # Check SLO violation - if duration > 0.5: # P99 SLO: 500ms - error_budget.labels(service='api', slo_type='latency_p99').inc() -``` - -**PromQL SLO Queries**: - -```promql -# API P95 latency -histogram_quantile(0.95, rate(api_request_duration_seconds_bucket[5m])) - -# SLO violation rate -rate(api_request_duration_seconds_bucket{le="0.2"}[5m]) / -rate(api_request_duration_seconds_count[5m]) - -# Error budget remaining -1 - (rate(slo_error_budget_consumed[1h]) / 3600) +async def handle(request): + async with sem: + return await do_work(request) ``` -#### Track SLOs via Prometheus/Grafana - -**Grafana Dashboard**: - -```json -{ - "dashboard": { - "panels": [ - { - "title": "API Latency SLO", - "targets": [ - { - "expr": "histogram_quantile(0.95, rate(api_request_duration_seconds_bucket[5m]))", - "legendFormat": "P95 Latency" - } - ], - "thresholds": [ - { - "value": 0.2, - "colorMode": "critical", - "fill": true - } - ] - } - ] - } -} -``` - -### Autoscaling Strategies - -#### HPA/VPA for RKE2 - -**Horizontal Pod Autoscaler**: - -```yaml -# k8s/hpa.yaml -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: api-hpa -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: api-service - minReplicas: 2 - maxReplicas: 10 - metrics: - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: 70 - - type: Resource - resource: - name: memory - target: - type: Utilization - averageUtilization: 80 - behavior: - scaleDown: - stabilizationWindowSeconds: 300 - policies: - - type: Percent - value: 50 - periodSeconds: 60 - scaleUp: - stabilizationWindowSeconds: 0 - policies: - - type: Percent - value: 100 - periodSeconds: 15 -``` - -**Vertical Pod Autoscaler**: - -```yaml -# k8s/vpa.yaml -apiVersion: autoscaling.k8s.io/v1 -kind: VerticalPodAutoscaler -metadata: - name: api-vpa -spec: - targetRef: - apiVersion: apps/v1 - kind: Deployment - name: api-service - updatePolicy: - updateMode: "Auto" - resourcePolicy: - containerPolicies: - - containerName: api - minAllowed: - cpu: 100m - memory: 128Mi - maxAllowed: - cpu: 2 - memory: 4Gi -``` - -#### CPU/GPU Autoscaling for Inference Workers - -**GPU Autoscaling**: - -```yaml -# k8s/gpu-hpa.yaml -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: ml-inference-hpa -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: ml-inference - minReplicas: 1 - maxReplicas: 5 - metrics: - - type: Pods - pods: - metric: - name: gpu_utilization - target: - type: AverageValue - averageValue: "70" -``` - -#### Autoscaling Postgres Read Replicas - -**Pattern**: Scale read replicas based on query load. - -```python -# autoscaling/postgres_replicas.py -class PostgresReplicaAutoscaler: - def __init__(self, k8s_client, deployment_name: str): - self.k8s = k8s_client - self.deployment_name = deployment_name - - async def scale_based_on_load(self): - """Scale read replicas based on query load""" - # Get current metrics - query_rate = await get_prometheus_metric( - "rate(postgres_queries_total[5m])" - ) - - # Calculate desired replicas - target_query_rate_per_replica = 1000 # queries/sec - desired_replicas = max( - 1, - int(query_rate / target_query_rate_per_replica) - ) - - # Scale deployment - await self.k8s.scale_deployment( - self.deployment_name, - replicas=desired_replicas - ) -``` - -#### Autoscaling Redis Clusters - -**Redis Cluster Autoscaling**: - -```yaml -# k8s/redis-hpa.yaml -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: redis-hpa -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: StatefulSet - name: redis - minReplicas: 3 - maxReplicas: 10 - metrics: - - type: Resource - resource: - name: memory - target: - type: Utilization - averageUtilization: 80 -``` - -#### Autoscale-on-Queue-Depth - -**Redis Streams Autoscaling**: - -```python -# autoscaling/queue_depth.py -class QueueDepthAutoscaler: - def __init__(self, k8s_client, deployment_name: str): - self.k8s = k8s_client - self.deployment_name = deployment_name - - async def scale_based_on_queue(self): - """Scale based on queue depth""" - # Get queue depth - queue_depth = await get_redis_stream_length("jobs:queue") - - # Calculate desired replicas (1 worker per 100 queued jobs) - jobs_per_worker = 100 - desired_replicas = max( - 1, - int(queue_depth / jobs_per_worker) - ) - - # Scale deployment - await self.k8s.scale_deployment( - self.deployment_name, - replicas=desired_replicas - ) -``` - -### Graceful Degradation Patterns - -#### Serving Stale Cached Responses - -**Pattern**: Return stale cache when service is down. - -```python -# graceful_degradation/stale_cache.py -class StaleCacheDegradation: - def __init__(self, cache_client: redis.Redis, max_staleness: int = 3600): - self.cache = cache_client - self.max_staleness = max_staleness - - async def get_with_stale_fallback(self, key: str, fetch_func: callable): - """Get value, return stale cache if fetch fails""" - try: - # Try fresh fetch - return await fetch_func() - except Exception: - # Fallback to stale cache - cached = self.cache.get(key) - if cached: - data = json.loads(cached) - age = time.time() - data.get("timestamp", 0) - - if age < self.max_staleness: - return data["value"] - - raise -``` - -#### Lower-Resolution Tiles - -**Pattern**: Return lower zoom tiles when under load. - -```python -# graceful_degradation/tile_degradation.py -class TileDegradation: - async def get_tile(self, z: int, x: int, y: int): - """Get tile, degrade resolution if under load""" - current_load = await get_system_load() - - if current_load > 0.9: - # Return lower zoom tile - if z > 10: - return await self.get_tile(z - 1, x // 2, y // 2) - - return await generate_tile(z, x, y) -``` - -#### Approximate Results - -**Pattern**: Return approximate results when exact computation is too slow. - -```python -# graceful_degradation/approximate.py -class ApproximateDegradation: - async def compute_result(self, query: str, exact: bool = True) -> dict: - """Compute result, return approximate if needed""" - start_time = time.time() - timeout = 5.0 - - try: - if exact: - result = await asyncio.wait_for( - compute_exact(query), - timeout=timeout - ) - else: - result = await compute_approximate(query) - - return result - except asyncio.TimeoutError: - # Fallback to approximate - return await compute_approximate(query) -``` - -#### Reduced ML Inference Precision - -**Pattern**: Use quantized models when under load. - -```python -# graceful_degradation/ml_precision.py -class MLPrecisionDegradation: - def __init__(self): - self.full_model = load_model("models/full.onnx") - self.quantized_model = load_model("models/quantized.onnx") - - async def infer(self, input_data: np.ndarray, current_load: float): - """Run inference, use quantized model if under load""" - if current_load > 0.9: - # Use quantized model (faster, lower precision) - return await self.quantized_model.infer(input_data) - else: - # Use full precision model - return await self.full_model.infer(input_data) -``` - -## GIS-Specific Resilience Patterns - -### Caching Geospatial Queries - -**Pattern**: Cache expensive spatial queries. - -```python -# gis_resilience/cache_geo_queries.py -class GeospatialQueryCache: - def __init__(self, redis_client: redis.Redis): - self.redis = redis_client - - async def execute_cached(self, query: str, params: dict) -> list: - """Execute geospatial query with caching""" - cache_key = f"gis:query:{hashlib.sha256(json.dumps(params).encode()).hexdigest()}" - - # Check cache - cached = self.redis.get(cache_key) - if cached: - return json.loads(cached) - - # Execute query - result = await execute_spatial_query(query, params) - - # Cache result - self.redis.setex(cache_key, 3600, json.dumps(result)) - - return result -``` - -### Precomputing Tiles as Fallback - -**Pattern**: Precompute tiles for common areas. - -```python -# gis_resilience/precompute_tiles.py -class TilePrecomputation: - async def precompute_common_tiles(self, bbox: dict, zoom_levels: list): - """Precompute tiles for common areas""" - for z in zoom_levels: - for x, y in generate_tile_coords(bbox, z): - tile = await generate_tile(z, x, y) - cache_key = f"gis:tile:{z}:{x}:{y}" - self.redis.setex(cache_key, 604800, tile.to_bytes()) -``` - -### Limiting Heavy ST_* Geoprocessing Functions - -**Pattern**: Limit concurrent spatial operations. - -```python -# gis_resilience/limit_geoprocessing.py -class GeoprocessingLimiter: - def __init__(self, max_concurrent: int = 4): - self.semaphore = asyncio.Semaphore(max_concurrent) - - async def st_intersects(self, geom1, geom2): - """ST_Intersects with concurrency limit""" - async with self.semaphore: - return await execute_st_intersects(geom1, geom2) -``` - -### Queuing Isochrone Requests - -**Pattern**: Queue isochrone requests to prevent overload. - -```python -# gis_resilience/isochrone_queue.py -class IsochroneQueue: - def __init__(self, max_queue: int = 100): - self.queue = asyncio.Queue(maxsize=max_queue) - self.worker_pool = AsyncWorkerPool(num_workers=2, queue_size=max_queue) - - async def compute_isochrone(self, point: dict, max_time: int) -> dict: - """Compute isochrone with queuing""" - future = asyncio.Future() - - try: - await asyncio.wait_for( - self.queue.put((point, max_time, future)), - timeout=0.1 - ) - return await future - except asyncio.TimeoutError: - raise QueueFullError("Isochrone queue full") -``` - -## ML-Specific Resilience Patterns - -### Concurrency-Safe ONNX Inference - -**Pattern**: Use thread-safe ONNX session. - -```python -# ml_resilience/onnx_concurrency.py -import threading - -class ThreadSafeONNXInference: - def __init__(self, model_path: str): - self.session = ort.InferenceSession(model_path) - self.lock = threading.Lock() - - def infer(self, input_data: np.ndarray) -> np.ndarray: - """Thread-safe ONNX inference""" - with self.lock: - return self.session.run(None, {"input": input_data}) -``` - -### Preloading Models into Memory - -**Pattern**: Preload models to avoid cold starts. - -```python -# ml_resilience/model_preload.py -class ModelPreloader: - def __init__(self, model_paths: list): - self.models = {} - self.loading = {} - - async def preload_models(self): - """Preload all models""" - for model_id, path in model_paths.items(): - asyncio.create_task(self._load_model(model_id, path)) - - async def _load_model(self, model_id: str, path: str): - """Load model asynchronously""" - self.loading[model_id] = True - try: - self.models[model_id] = ort.InferenceSession(path) - finally: - self.loading[model_id] = False -``` - -### Hot-Path ML Fallback Models - -**Pattern**: Use smaller models when under load. - -```python -# ml_resilience/fallback_models.py -class MLFallbackModels: - def __init__(self): - self.full_model = load_model("models/full.onnx") - self.fallback_model = load_model("models/fallback.onnx") - - async def infer(self, input_data: np.ndarray, current_load: float): - """Run inference with fallback model if needed""" - if current_load > 0.9: - return await self.fallback_model.infer(input_data) - return await self.full_model.infer(input_data) -``` - -## Architecture Diagrams - -### Concurrency Control Reference - -```mermaid -graph TB - subgraph Clients["Clients"] - C1["Client 1"] - C2["Client 2"] - C3["Client N"] - end - - subgraph RateLimit["Rate Limiter"] - TB["Token Bucket
(100 req/s)"] - end - - subgraph Concurrency["Concurrency Control"] - Sem["Semaphore
(Max 10)"] - Queue["Queue
(Max 100)"] - end - - subgraph Workers["Worker Pools"] - WP1["User-Facing Pool
(10 workers)"] - WP2["ETL Pool
(5 workers)"] - WP3["ML Pool
(2 workers)"] - end - - subgraph Services["Services"] - API["API Service"] - PG["Postgres
(Pool: 20)"] - ML["ML Service"] - end - - C1 & C2 & C3 --> TB - TB --> Sem - Sem --> Queue - Queue --> WP1 & WP2 & WP3 - WP1 --> API - WP2 --> PG - WP3 --> ML -``` - -### Rate Limit Gateway - -```mermaid -graph LR - Client["Client"] --> NGINX["NGINX
(Rate Limit Zone)"] - NGINX -->|"10 req/s"| API["API Service"] - API -->|"Per-User Limit"| Redis["Redis
(Token Bucket)"] - Redis -->|"Allowed"| Backend["Backend Services"] - Redis -->|"Rate Limited"| Reject["429 Too Many Requests"] -``` - -## Observability & Telemetry - -### PromQL Queries - -```promql -# Queue depth -rate(queue_size[5m]) - -# Request latency percentiles -histogram_quantile(0.95, rate(request_duration_seconds_bucket[5m])) - -# Circuit breaker state -circuit_breaker_state{state="open"} - -# Rate limit hits -rate(rate_limit_hits_total[5m]) - -# Backpressure rejections -rate(backpressure_rejections_total[5m]) -``` - -## Anti-Patterns - -### Unlimited Concurrency - -**Symptom**: System overwhelmed, OOM errors, service crashes. - -**Fix**: Always set concurrency limits (semaphores, worker pools). - -### No Timeout or Retry Limits - -**Symptom**: Requests hang indefinitely, resources exhausted. - -**Fix**: Set timeouts on all operations, limit retries with exponential backoff. - -### Retry Storms Meltdown - -**Symptom**: Cascading failures from unlimited retries. - -**Fix**: Use circuit breakers, exponential backoff, jitter, retry budgets. - -### DB Saturated Due to "Pool=All" Connections - -**Symptom**: Database connection exhaustion, all requests fail. - -**Fix**: Use connection pools with limits, monitor pool usage. - -### ML Inference Workers Blocking for GPU Warmup - -**Symptom**: All workers blocked waiting for GPU, queue builds up. - -**Fix**: Preload models, use separate warmup workers, graceful degradation. - -### GIS Queries Running Unbounded - -**Symptom**: Expensive spatial queries block all requests. - -**Fix**: Limit concurrent spatial operations, queue expensive queries, use timeouts. - -### Redis-Backed Rate Limiting with No TTLs - -**Symptom**: Rate limit keys never expire, memory leak. - -**Fix**: Always set TTL on rate limit keys, use sliding windows. - -### Using sleep() Instead of Proper Backpressure - -**Symptom**: Inefficient resource usage, poor responsiveness. - -**Fix**: Use semaphores, queues, proper async/await patterns. - -### Running All Workloads on Single Worker Class - -**Symptom**: One workload type blocks all others. - -**Fix**: Use bulkheads, separate worker pools per workload type. - -### No Circuit Breaking on Remote Services - -**Symptom**: Cascading failures when remote service is down. - -**Fix**: Implement circuit breakers for all remote service calls. - -### Infinite Queue Growth - -**Symptom**: Memory exhaustion from unbounded queues. +## Backpressure -**Fix**: Set queue size limits, implement backpressure, reject when full. +Backpressure means **slowing producers** when consumers fall behind. -### Coupling Retries Across Microservices +Mechanisms I actually use: -**Symptom**: Retry storms cascade across services. +1. **Bounded queues** — `maxsize` on asyncio queues, channel buffers in Go, Redis stream consumer lag alerts. +2. **HTTP 503 + Retry-After** — when the service is saturated, reject early instead of timing out deep inside the stack. +3. **Admission control at the edge** — API gateway or load balancer max connections; rate limits per tenant. +4. **Database-side guardrails** — `statement_timeout`, connection pool limits, read replicas for read-heavy paths. -**Fix**: Use circuit breakers, limit retries, use exponential backoff with jitter. +Anti-pattern: unbounded in-memory queues “because we will scale workers later.” That is how you OOM during a traffic spike. -## Final Checklists +## Rate limiting -### Rate Limiting Readiness +Use rate limits to protect shared resources (auth endpoints, expensive GIS exports, model inference). -- [ ] Rate limits defined for all endpoints -- [ ] Per-user, per-IP, per-org limits configured -- [ ] Redis-based rate limiting with TTLs -- [ ] NGINX rate limits configured -- [ ] Rate limit monitoring in place -- [ ] Rate limit error handling implemented +- **Token bucket** — smooth bursts with a sustained average (good for user-facing APIs). +- **Fixed window counters** — simple in Redis; watch boundary effects at window rollovers. +- **Per-tenant keys** — avoid one noisy neighbor taking the whole service. -### Concurrency Safety +Do not copy numeric limits from blog posts. Derive them from **measured** capacity and SLOs you actually monitor. -- [ ] Concurrency limits set for all pools -- [ ] CPU-bound vs IO-bound workloads separated -- [ ] Worker pools sized appropriately -- [ ] Semaphores used for limiting -- [ ] Thread safety verified -- [ ] Deadlock prevention measures +## Load shedding and degradation -### Backpressure Correctness +When overloaded: -- [ ] Queue size limits set -- [ ] Backpressure signals implemented -- [ ] Rejection handling in place -- [ ] Queue depth monitored -- [ ] Backpressure propagated upstream +1. Drop or defer **optional** work first (recommendations, previews, non-critical tiles). +2. Serve **cached** or **coarser** results (lower zoom, simplified geometry, stale read). +3. Fail fast on **new** work while draining in-flight requests. -### Load Shedding +Circuit breakers help when a dependency is **known bad** (repeated errors, health check failing). They are not a substitute for pool sizing. -- [ ] Priority levels defined -- [ ] Shedding thresholds configured -- [ ] Low-priority traffic identified -- [ ] Graceful degradation implemented -- [ ] Shedding metrics tracked +## Observability (minimum) -### Circuit Breaking +Before tuning limits, you need: -- [ ] Circuit breakers for all remote calls -- [ ] Failure thresholds configured -- [ ] Timeout values set -- [ ] Half-open state handling -- [ ] Fallback behaviors defined +- In-flight request count / queue depth +- Pool utilization (DB, Redis, HTTP client) +- p95/p99 latency **per dependency**, not just the edge +- Error rate split by timeout vs 4xx vs 5xx -### Autoscaling +Prometheus/Grafana/Loki are fine; the point is to see **saturation**, not only CPU. -- [ ] HPA configured for services -- [ ] Metrics-based scaling rules -- [ ] Queue-depth scaling implemented -- [ ] Scale-down policies configured -- [ ] Resource limits set +## GIS-specific notes -### Observability Coverage +- Cap concurrent **tile** or **WFS** exports; large bounding boxes are abuse vectors. +- Prefer **pre-tiled** or **materialized** layers for hot paths; do not let ad hoc `ST_Union` on millions of rows run concurrently without a queue. +- Index discipline (see [PostGIS geometry indexing tutorial](../../tutorials/database-data-engineering/postgis-geometry-indexing.md)) matters more than micro-optimizing thread counts. -- [ ] Latency percentiles tracked -- [ ] Queue depths monitored -- [ ] Circuit breaker states exposed -- [ ] Rate limit hits logged -- [ ] Error budgets tracked -- [ ] SLO dashboards configured +## ML inference notes -## See Also +- Bound batch size and queue depth on GPU workers. +- Separate **online** inference from **batch** scoring where possible. +- Fallback to a smaller model only if you have tested quality impact — not as a default config copied from a diagram. -- **[Performance Best Practices](../performance/end-to-end-caching-strategy.md)** - Caching strategies -- **[Grafana, Prometheus, Loki, and Observability](grafana-prometheus-loki-observability.md)** - Observability patterns -- **[Testing Best Practices](testing-best-practices.md)** - Resilience testing +## What I distrust ---- +- Pages that list twelve languages and fifteen patterns without sizing guidance. +- “Production-ready” sample code that never mentions pool limits or cancellation. +- SLO tables with precise percentages and no measurement method. +- Autoscaling rules copied from hyperscaler docs onto a three-node lab cluster. -*This guide provides a complete framework for system resilience. Start with rate limiting and concurrency control, then add circuit breakers and backpressure. The goal is graceful degradation under load, not perfect availability.* +## Further reading +- [Failure-oriented system design](../operations/failure-oriented-design.md) +- [Configuration management](configuration-management.md) +- PostgreSQL: [connection pooling tutorial](../../tutorials/database-data-engineering/postgres-pooling.md) +- Google SRE Book — overload and cascading failures (external) diff --git a/docs/best-practices/operations-monitoring/unified-observability-architecture.md b/docs/best-practices/operations-monitoring/unified-observability-architecture.md index 2a9f5c6..36b3a4e 100644 --- a/docs/best-practices/operations-monitoring/unified-observability-architecture.md +++ b/docs/best-practices/operations-monitoring/unified-observability-architecture.md @@ -1,6 +1,5 @@ # Observability as Architecture: Unified Telemetry Models Across Clusters, Services, and Languages -**Objective**: Establish a unified telemetry model across all systems, services, and languages. When you need structured logging, when you want distributed tracing, when you need real-time dashboards—this guide provides the complete framework. ## Introduction diff --git a/docs/best-practices/operations/failure-oriented-design.md b/docs/best-practices/operations/failure-oriented-design.md index dcc9e46..9ff568f 100644 --- a/docs/best-practices/operations/failure-oriented-design.md +++ b/docs/best-practices/operations/failure-oriented-design.md @@ -1,6 +1,5 @@ # Failure-Oriented System Design -**Objective**: Design systems assuming failure is inevitable—so that when components fail, the system degrades predictably and recovers without cascading damage. ## Failure as the default state diff --git a/docs/best-practices/operations/system-resilience-and-concurrency.md b/docs/best-practices/operations/system-resilience-and-concurrency.md index 3a9d440..f3e8ce3 100644 --- a/docs/best-practices/operations/system-resilience-and-concurrency.md +++ b/docs/best-practices/operations/system-resilience-and-concurrency.md @@ -1,6 +1,5 @@ # System Resilience, Rate Limiting, Concurrency Control & Backpressure: Best Practices for Distributed Systems -**Objective**: Master production-grade resilience patterns for distributed systems. When you need to prevent cascading failures, control concurrency, implement rate limiting, handle backpressure, and maintain SLOs under load—this guide provides complete patterns and implementations. ## Introduction diff --git a/docs/best-practices/performance/end-to-end-caching-strategy.md b/docs/best-practices/performance/end-to-end-caching-strategy.md index 172fb99..b0c9eec 100644 --- a/docs/best-practices/performance/end-to-end-caching-strategy.md +++ b/docs/best-practices/performance/end-to-end-caching-strategy.md @@ -1,6 +1,5 @@ # End-to-End Caching Strategy & Performance Layering: Best Practices for High-Performance Distributed Systems -**Objective**: Master production-grade caching strategies across multi-layer distributed systems. When you need to optimize API performance, accelerate ETL workloads, reduce ML inference latency, improve GIS tile delivery, and eliminate database bottlenecks—this guide provides complete caching patterns and governance. ## Introduction diff --git a/docs/best-practices/performance/index.md b/docs/best-practices/performance/index.md index 7ff55c2..61171a1 100644 --- a/docs/best-practices/performance/index.md +++ b/docs/best-practices/performance/index.md @@ -1,6 +1,5 @@ # Performance Best Practices -**Objective**: Master production-grade performance optimization for distributed systems. When you need to optimize API latency, accelerate ETL workloads, reduce ML inference time, improve GIS tile delivery, and eliminate database bottlenecks—these performance best practices become your foundation. This collection provides comprehensive guides for caching strategies, performance optimization, and system tuning. Each guide includes architectural patterns, configuration examples, and real-world implementation strategies. diff --git a/docs/best-practices/postgres/fdw-postgres.md b/docs/best-practices/postgres/fdw-postgres.md index ff2b330..076ec7a 100644 --- a/docs/best-practices/postgres/fdw-postgres.md +++ b/docs/best-practices/postgres/fdw-postgres.md @@ -1,6 +1,5 @@ # Postgres FDW Best Practices -**Objective**: FDWs make Postgres the query engine for your data lake — but misuse them, and you'll summon performance demons. Here's how to tame them. FDWs make Postgres the query engine for your data lake — but misuse them, and you'll summon performance demons. Here's how to tame them. @@ -992,4 +991,3 @@ FDWs require understanding external data sources, performance implications, and --- -*This guide provides the complete machinery for Postgres FDWs. The patterns scale from simple parquet files to complex cloud data lakes, from basic queries to advanced geospatial federated analytics.* diff --git a/docs/best-practices/postgres/index.md b/docs/best-practices/postgres/index.md index 5f5d4d6..a05b071 100644 --- a/docs/best-practices/postgres/index.md +++ b/docs/best-practices/postgres/index.md @@ -1,6 +1,5 @@ # PostgreSQL Best Practices -**Objective**: Master senior-level PostgreSQL patterns for production systems. When you need to optimize database performance, when you want to implement robust data management strategies, when you need enterprise-grade PostgreSQL solutions—these best practices become your weapon of choice. ## Core Development @@ -52,4 +51,3 @@ --- -*This guide provides the complete machinery for PostgreSQL excellence. Each pattern includes implementation examples, optimization strategies, and real-world usage patterns for enterprise PostgreSQL systems.* diff --git a/docs/best-practices/postgres/postgis-best-practices.md b/docs/best-practices/postgres/postgis-best-practices.md index 0ffe295..b16188a 100644 --- a/docs/best-practices/postgres/postgis-best-practices.md +++ b/docs/best-practices/postgres/postgis-best-practices.md @@ -6,7 +6,6 @@ tags: # PostGIS Best Practices -**Objective**: Master senior-level PostGIS spatial data patterns for production systems. When you need to handle geospatial data, when you want to optimize spatial queries, when you need enterprise-grade spatial analysis strategies—these best practices become your weapon of choice. ## Core Principles @@ -722,7 +721,6 @@ def setup_postgis_spatial_data(): --- -*This guide provides the complete machinery for PostGIS spatial data excellence. Each pattern includes implementation examples, spatial analysis strategies, and real-world usage patterns for enterprise PostGIS spatial systems.* !!! tip "See also" - [PostGIS Geometry Indexing Tutorial](../../tutorials/database-data-engineering/postgis-geometry-indexing.md) — hands-on implementation of the indexing patterns from this guide diff --git a/docs/best-practices/postgres/postgres-api-development.md b/docs/best-practices/postgres/postgres-api-development.md index 3dd21e2..bab1efd 100644 --- a/docs/best-practices/postgres/postgres-api-development.md +++ b/docs/best-practices/postgres/postgres-api-development.md @@ -1,6 +1,5 @@ # PostgreSQL API Development Best Practices -**Objective**: Master senior-level PostgreSQL API development patterns for production systems. When you need to build robust APIs with PostgreSQL, when you want to implement RESTful services, when you need enterprise-grade API strategies—these best practices become your weapon of choice. ## Core Principles @@ -709,4 +708,3 @@ def setup_postgresql_api_development(): --- -*This guide provides the complete machinery for PostgreSQL API development excellence. Each pattern includes implementation examples, API strategies, and real-world usage patterns for enterprise PostgreSQL API systems.* diff --git a/docs/best-practices/postgres/postgres-backup-recovery.md b/docs/best-practices/postgres/postgres-backup-recovery.md index 980d95a..35b7e09 100644 --- a/docs/best-practices/postgres/postgres-backup-recovery.md +++ b/docs/best-practices/postgres/postgres-backup-recovery.md @@ -1,6 +1,5 @@ # PostgreSQL Backup & Recovery Best Practices -**Objective**: Master senior-level PostgreSQL backup and recovery strategies for production systems. When you need to implement robust backup solutions, when you want to ensure data protection, when you need enterprise-grade recovery strategies—these best practices become your weapon of choice. ## Core Principles @@ -731,4 +730,3 @@ def setup_postgresql_backup_recovery(): --- -*This guide provides the complete machinery for PostgreSQL backup and recovery excellence. Each pattern includes implementation examples, recovery strategies, and real-world usage patterns for enterprise PostgreSQL backup systems.* diff --git a/docs/best-practices/postgres/postgres-cloud-integration.md b/docs/best-practices/postgres/postgres-cloud-integration.md index 144aedd..71ae321 100644 --- a/docs/best-practices/postgres/postgres-cloud-integration.md +++ b/docs/best-practices/postgres/postgres-cloud-integration.md @@ -1,6 +1,5 @@ # PostgreSQL Cloud Integration Best Practices -**Objective**: Master senior-level PostgreSQL cloud integration patterns for production systems. When you need to integrate PostgreSQL with cloud services, when you want to implement cloud-native patterns, when you need enterprise-grade cloud strategies—these best practices become your weapon of choice. ## Core Principles @@ -796,4 +795,3 @@ def setup_postgresql_cloud_integration(): --- -*This guide provides the complete machinery for PostgreSQL cloud integration excellence. Each pattern includes implementation examples, cloud strategies, and real-world usage patterns for enterprise PostgreSQL cloud systems.* diff --git a/docs/best-practices/postgres/postgres-configuration-management.md b/docs/best-practices/postgres/postgres-configuration-management.md index 4f5da73..9c437c0 100644 --- a/docs/best-practices/postgres/postgres-configuration-management.md +++ b/docs/best-practices/postgres/postgres-configuration-management.md @@ -1,6 +1,5 @@ # PostgreSQL Configuration Management Best Practices -**Objective**: Master senior-level PostgreSQL configuration management patterns for production systems. When you need to manage database configurations, when you want to implement configuration as code, when you need enterprise-grade configuration strategies—these best practices become your weapon of choice. ## Core Principles @@ -675,4 +674,3 @@ def setup_postgresql_configuration_management(): --- -*This guide provides the complete machinery for PostgreSQL configuration management excellence. Each pattern includes implementation examples, configuration strategies, and real-world usage patterns for enterprise PostgreSQL configuration systems.* diff --git a/docs/best-practices/postgres/postgres-constraints-validation.md b/docs/best-practices/postgres/postgres-constraints-validation.md index 6ee978b..cc59983 100644 --- a/docs/best-practices/postgres/postgres-constraints-validation.md +++ b/docs/best-practices/postgres/postgres-constraints-validation.md @@ -1,6 +1,5 @@ # PostgreSQL Constraints & Validation Best Practices -**Objective**: Master senior-level PostgreSQL constraint and validation patterns for production systems. When you need to enforce data integrity, when you want to implement robust validation, when you need enterprise-grade constraint strategies—these best practices become your weapon of choice. ## Core Principles @@ -718,4 +717,3 @@ def setup_postgresql_constraints_validation(): --- -*This guide provides the complete machinery for PostgreSQL constraints and validation excellence. Each pattern includes implementation examples, validation strategies, and real-world usage patterns for enterprise PostgreSQL constraint systems.* diff --git a/docs/best-practices/postgres/postgres-containerization.md b/docs/best-practices/postgres/postgres-containerization.md index ce6d328..6809db6 100644 --- a/docs/best-practices/postgres/postgres-containerization.md +++ b/docs/best-practices/postgres/postgres-containerization.md @@ -1,6 +1,5 @@ # PostgreSQL Containerization Best Practices -**Objective**: Master senior-level PostgreSQL containerization patterns for production systems. When you need to containerize PostgreSQL, when you want to implement Docker best practices, when you need enterprise-grade container strategies—these best practices become your weapon of choice. ## Core Principles @@ -878,4 +877,3 @@ def setup_postgresql_containerization(): --- -*This guide provides the complete machinery for PostgreSQL containerization excellence. Each pattern includes implementation examples, container strategies, and real-world usage patterns for enterprise PostgreSQL containerized systems.* diff --git a/docs/best-practices/postgres/postgres-data-pipeline-integration.md b/docs/best-practices/postgres/postgres-data-pipeline-integration.md index 7ddd008..9b5fe2b 100644 --- a/docs/best-practices/postgres/postgres-data-pipeline-integration.md +++ b/docs/best-practices/postgres/postgres-data-pipeline-integration.md @@ -1,6 +1,5 @@ # PostgreSQL Data Pipeline Integration Best Practices -**Objective**: Master senior-level PostgreSQL data pipeline patterns for production systems. When you need to integrate PostgreSQL with data pipelines, when you want to implement ETL workflows, when you need enterprise-grade data processing—these best practices become your weapon of choice. ## Core Principles @@ -876,4 +875,3 @@ def setup_postgresql_data_pipeline_integration(): --- -*This guide provides the complete machinery for PostgreSQL data pipeline integration excellence. Each pattern includes implementation examples, pipeline strategies, and real-world usage patterns for enterprise PostgreSQL data pipeline systems.* diff --git a/docs/best-practices/postgres/postgres-data-types.md b/docs/best-practices/postgres/postgres-data-types.md index 5788f91..8b1c094 100644 --- a/docs/best-practices/postgres/postgres-data-types.md +++ b/docs/best-practices/postgres/postgres-data-types.md @@ -1,6 +1,5 @@ # PostgreSQL Data Types Best Practices -**Objective**: Master senior-level PostgreSQL data type patterns for production systems. When you need to choose optimal data types, when you want to optimize storage and performance, when you need enterprise-grade data type strategies—these best practices become your weapon of choice. ## Core Principles @@ -832,4 +831,3 @@ def setup_postgresql_data_types(): --- -*This guide provides the complete machinery for PostgreSQL data types excellence. Each pattern includes implementation examples, optimization strategies, and real-world usage patterns for enterprise PostgreSQL data type systems.* diff --git a/docs/best-practices/postgres/postgres-database-design.md b/docs/best-practices/postgres/postgres-database-design.md index 9780da0..78d839a 100644 --- a/docs/best-practices/postgres/postgres-database-design.md +++ b/docs/best-practices/postgres/postgres-database-design.md @@ -1,6 +1,5 @@ # PostgreSQL Database Design Best Practices -**Objective**: Master senior-level PostgreSQL database design patterns for production systems. When you need to design efficient schemas, when you want to optimize data relationships, when you need enterprise-grade database design strategies—these best practices become your weapon of choice. ## Core Principles @@ -706,4 +705,3 @@ def setup_postgresql_database_design(): --- -*This guide provides the complete machinery for PostgreSQL database design excellence. Each pattern includes implementation examples, design strategies, and real-world usage patterns for enterprise PostgreSQL database systems.* diff --git a/docs/best-practices/postgres/postgres-deployment-strategies.md b/docs/best-practices/postgres/postgres-deployment-strategies.md index 21ace61..7465bd2 100644 --- a/docs/best-practices/postgres/postgres-deployment-strategies.md +++ b/docs/best-practices/postgres/postgres-deployment-strategies.md @@ -1,6 +1,5 @@ # PostgreSQL Deployment Strategies Best Practices -**Objective**: Master senior-level PostgreSQL deployment patterns for production systems. When you need to deploy PostgreSQL at scale, when you want to implement zero-downtime deployments, when you need enterprise-grade deployment strategies—these best practices become your weapon of choice. ## Core Principles @@ -653,4 +652,3 @@ def setup_postgresql_deployment(): --- -*This guide provides the complete machinery for PostgreSQL deployment excellence. Each pattern includes implementation examples, deployment strategies, and real-world usage patterns for enterprise PostgreSQL deployment systems.* diff --git a/docs/best-practices/postgres/postgres-dev-environment.md b/docs/best-practices/postgres/postgres-dev-environment.md index be84222..1488d6b 100644 --- a/docs/best-practices/postgres/postgres-dev-environment.md +++ b/docs/best-practices/postgres/postgres-dev-environment.md @@ -1,6 +1,5 @@ # PostgreSQL Development Environment Best Practices -**Objective**: Master senior-level PostgreSQL development environment setup for production systems. When you need to set up a robust development environment, when you want to optimize your PostgreSQL workflow, when you need enterprise-grade development patterns—these best practices become your weapon of choice. ## Core Principles @@ -586,4 +585,3 @@ def setup_postgresql_development(): --- -*This guide provides the complete machinery for PostgreSQL development excellence. Each pattern includes implementation examples, configuration strategies, and real-world usage patterns for enterprise PostgreSQL development systems.* diff --git a/docs/best-practices/postgres/postgres-event-driven.md b/docs/best-practices/postgres/postgres-event-driven.md index 1ddcaad..d4b0ba2 100644 --- a/docs/best-practices/postgres/postgres-event-driven.md +++ b/docs/best-practices/postgres/postgres-event-driven.md @@ -1,6 +1,5 @@ # PostgreSQL Event-Driven Architecture Best Practices -**Objective**: Master senior-level PostgreSQL event-driven patterns for production systems. When you need to implement event-driven architectures, when you want to build reactive systems, when you need enterprise-grade event processing—these best practices become your weapon of choice. ## Core Principles @@ -800,4 +799,3 @@ def setup_postgresql_event_driven(): --- -*This guide provides the complete machinery for PostgreSQL event-driven architecture excellence. Each pattern includes implementation examples, event processing strategies, and real-world usage patterns for enterprise PostgreSQL event-driven systems.* diff --git a/docs/best-practices/postgres/postgres-extensions.md b/docs/best-practices/postgres/postgres-extensions.md index b8fd351..5747a4c 100644 --- a/docs/best-practices/postgres/postgres-extensions.md +++ b/docs/best-practices/postgres/postgres-extensions.md @@ -1,6 +1,5 @@ # PostgreSQL Extensions Best Practices -**Objective**: Master senior-level PostgreSQL extension patterns for production systems. When you need to extend PostgreSQL functionality, when you want to leverage specialized capabilities, when you need enterprise-grade extension strategies—these best practices become your weapon of choice. ## Core Principles @@ -702,4 +701,3 @@ def setup_postgresql_extensions(): --- -*This guide provides the complete machinery for PostgreSQL extensions excellence. Each pattern includes implementation examples, extension strategies, and real-world usage patterns for enterprise PostgreSQL extension systems.* diff --git a/docs/best-practices/postgres/postgres-fulltext-search.md b/docs/best-practices/postgres/postgres-fulltext-search.md index 9a677dd..e02201c 100644 --- a/docs/best-practices/postgres/postgres-fulltext-search.md +++ b/docs/best-practices/postgres/postgres-fulltext-search.md @@ -1,6 +1,5 @@ # PostgreSQL Full-Text Search Best Practices -**Objective**: Master senior-level PostgreSQL full-text search patterns for production systems. When you need to implement powerful text search, when you want to optimize search performance, when you need enterprise-grade search strategies—these best practices become your weapon of choice. ## Core Principles @@ -748,4 +747,3 @@ def setup_postgresql_fulltext_search(): --- -*This guide provides the complete machinery for PostgreSQL full-text search excellence. Each pattern includes implementation examples, optimization strategies, and real-world usage patterns for enterprise PostgreSQL search systems.* diff --git a/docs/best-practices/postgres/postgres-indexing-strategies.md b/docs/best-practices/postgres/postgres-indexing-strategies.md index 93df853..0bcef41 100644 --- a/docs/best-practices/postgres/postgres-indexing-strategies.md +++ b/docs/best-practices/postgres/postgres-indexing-strategies.md @@ -1,6 +1,5 @@ # PostgreSQL Indexing Strategies Best Practices -**Objective**: Master senior-level PostgreSQL indexing patterns for production systems. When you need to optimize query performance, when you want to design efficient indexes, when you need enterprise-grade indexing strategies—these best practices become your weapon of choice. ## Core Principles @@ -676,4 +675,3 @@ def setup_postgresql_indexing(): --- -*This guide provides the complete machinery for PostgreSQL indexing excellence. Each pattern includes implementation examples, optimization strategies, and real-world usage patterns for enterprise PostgreSQL indexing systems.* diff --git a/docs/best-practices/postgres/postgres-json-jsonb.md b/docs/best-practices/postgres/postgres-json-jsonb.md index a486e72..311895f 100644 --- a/docs/best-practices/postgres/postgres-json-jsonb.md +++ b/docs/best-practices/postgres/postgres-json-jsonb.md @@ -1,6 +1,5 @@ # PostgreSQL JSON & JSONB Best Practices -**Objective**: Master senior-level PostgreSQL JSON and JSONB patterns for production systems. When you need to work with semi-structured data, when you want to leverage JSON capabilities, when you need enterprise-grade JSON strategies—these best practices become your weapon of choice. ## Core Principles @@ -658,4 +657,3 @@ def setup_postgresql_json_jsonb(): --- -*This guide provides the complete machinery for PostgreSQL JSON/JSONB excellence. Each pattern includes implementation examples, optimization strategies, and real-world usage patterns for enterprise PostgreSQL JSON systems.* diff --git a/docs/best-practices/postgres/postgres-large-objects.md b/docs/best-practices/postgres/postgres-large-objects.md index 9caaf21..18d3256 100644 --- a/docs/best-practices/postgres/postgres-large-objects.md +++ b/docs/best-practices/postgres/postgres-large-objects.md @@ -1,6 +1,5 @@ # PostgreSQL Large Object Storage Best Practices -**Objective**: Master senior-level PostgreSQL large object storage patterns for production systems. When you need to handle BLOBs, when you want to optimize large object storage, when you need enterprise-grade large object strategies—these best practices become your weapon of choice. ## Core Principles @@ -749,4 +748,3 @@ def setup_postgresql_large_objects(): --- -*This guide provides the complete machinery for PostgreSQL large object storage excellence. Each pattern includes implementation examples, storage strategies, and real-world usage patterns for enterprise PostgreSQL large object systems.* diff --git a/docs/best-practices/postgres/postgres-maintenance-vacuum.md b/docs/best-practices/postgres/postgres-maintenance-vacuum.md index 7a2cc60..6369613 100644 --- a/docs/best-practices/postgres/postgres-maintenance-vacuum.md +++ b/docs/best-practices/postgres/postgres-maintenance-vacuum.md @@ -1,6 +1,5 @@ # PostgreSQL Maintenance & Vacuum Best Practices -**Objective**: Master senior-level PostgreSQL maintenance and vacuum patterns for production systems. When you need to maintain database health, when you want to optimize vacuum operations, when you need enterprise-grade maintenance strategies—these best practices become your weapon of choice. ## Core Principles @@ -742,4 +741,3 @@ def setup_postgresql_maintenance_vacuum(): --- -*This guide provides the complete machinery for PostgreSQL maintenance and vacuum excellence. Each pattern includes implementation examples, maintenance strategies, and real-world usage patterns for enterprise PostgreSQL maintenance systems.* diff --git a/docs/best-practices/postgres/postgres-monitoring-observability.md b/docs/best-practices/postgres/postgres-monitoring-observability.md index a114d2e..1a3df93 100644 --- a/docs/best-practices/postgres/postgres-monitoring-observability.md +++ b/docs/best-practices/postgres/postgres-monitoring-observability.md @@ -1,6 +1,5 @@ # PostgreSQL Monitoring & Observability Best Practices -**Objective**: Master senior-level PostgreSQL monitoring and observability patterns for production systems. When you need to implement comprehensive monitoring, when you want to track performance metrics, when you need enterprise-grade observability strategies—these best practices become your weapon of choice. ## Core Principles @@ -806,4 +805,3 @@ def setup_postgresql_monitoring(): --- -*This guide provides the complete machinery for PostgreSQL monitoring excellence. Each pattern includes implementation examples, monitoring strategies, and real-world usage patterns for enterprise PostgreSQL monitoring systems.* diff --git a/docs/best-practices/postgres/postgres-partitioning.md b/docs/best-practices/postgres/postgres-partitioning.md index 01eee80..78e9222 100644 --- a/docs/best-practices/postgres/postgres-partitioning.md +++ b/docs/best-practices/postgres/postgres-partitioning.md @@ -1,6 +1,5 @@ # PostgreSQL Partitioning Best Practices -**Objective**: Master senior-level PostgreSQL partitioning strategies for production systems. When you need to manage large tables efficiently, when you want to improve query performance, when you need enterprise-grade partitioning strategies—these best practices become your weapon of choice. ## Core Principles @@ -679,4 +678,3 @@ def setup_postgresql_partitioning(): --- -*This guide provides the complete machinery for PostgreSQL partitioning excellence. Each pattern includes implementation examples, optimization strategies, and real-world usage patterns for enterprise PostgreSQL partitioning systems.* diff --git a/docs/best-practices/postgres/postgres-performance-tuning.md b/docs/best-practices/postgres/postgres-performance-tuning.md index 8576f65..bdae98b 100644 --- a/docs/best-practices/postgres/postgres-performance-tuning.md +++ b/docs/best-practices/postgres/postgres-performance-tuning.md @@ -1,6 +1,5 @@ # PostgreSQL Performance Tuning Best Practices -**Objective**: Master senior-level PostgreSQL performance optimization for production systems. When you need to optimize query performance, when you want to tune database configuration, when you need enterprise-grade performance strategies—these best practices become your weapon of choice. ## Core Principles @@ -645,4 +644,3 @@ def setup_postgresql_performance_tuning(): --- -*This guide provides the complete machinery for PostgreSQL performance excellence. Each pattern includes implementation examples, optimization strategies, and real-world usage patterns for enterprise PostgreSQL performance systems.* diff --git a/docs/best-practices/postgres/postgres-pooling.md b/docs/best-practices/postgres/postgres-pooling.md index a37a4b1..a070971 100644 --- a/docs/best-practices/postgres/postgres-pooling.md +++ b/docs/best-practices/postgres/postgres-pooling.md @@ -1,6 +1,5 @@ # PostgreSQL Connection Pooling Best Practices -**Objective**: Master production-grade PostgreSQL connection pooling for high-concurrency workloads. When you need to handle thousands of concurrent requests, when you're running FastAPI with geospatial queries, when you're managing Prefect agents and Dask workers—PostgreSQL connection pooling becomes your lifeline. Postgres will not forgive sloppy connection handling. Without pooling, you'll hit limits, crash services, and wonder why everything stalls. @@ -870,4 +869,3 @@ PostgreSQL connection pooling requires understanding connection limits, pooling --- -*This guide provides the complete machinery for PostgreSQL connection pooling. The patterns scale from simple applications to complex microservices, from basic pooling to advanced connection management.* diff --git a/docs/best-practices/postgres/postgres-replication-ha.md b/docs/best-practices/postgres/postgres-replication-ha.md index c2c100b..affcba7 100644 --- a/docs/best-practices/postgres/postgres-replication-ha.md +++ b/docs/best-practices/postgres/postgres-replication-ha.md @@ -1,6 +1,5 @@ # PostgreSQL Replication & High Availability Best Practices -**Objective**: Master senior-level PostgreSQL replication and high availability patterns for production systems. When you need to implement robust replication, when you want to ensure zero-downtime deployments, when you need enterprise-grade high availability strategies—these best practices become your weapon of choice. ## Core Principles @@ -775,4 +774,3 @@ def setup_postgresql_replication_ha(): --- -*This guide provides the complete machinery for PostgreSQL replication and high availability excellence. Each pattern includes implementation examples, HA strategies, and real-world usage patterns for enterprise PostgreSQL HA systems.* diff --git a/docs/best-practices/postgres/postgres-scaling-strategies.md b/docs/best-practices/postgres/postgres-scaling-strategies.md index 784db8c..acc1f3a 100644 --- a/docs/best-practices/postgres/postgres-scaling-strategies.md +++ b/docs/best-practices/postgres/postgres-scaling-strategies.md @@ -1,6 +1,5 @@ # PostgreSQL Scaling Strategies Best Practices -**Objective**: Master senior-level PostgreSQL scaling patterns for production systems. When you need to scale database performance, when you want to implement horizontal scaling, when you need enterprise-grade scaling strategies—these best practices become your weapon of choice. ## Core Principles @@ -759,4 +758,3 @@ def setup_postgresql_scaling(): --- -*This guide provides the complete machinery for PostgreSQL scaling excellence. Each pattern includes implementation examples, scaling strategies, and real-world usage patterns for enterprise PostgreSQL scaling systems.* diff --git a/docs/best-practices/postgres/postgres-security-best-practices.md b/docs/best-practices/postgres/postgres-security-best-practices.md index 6f1c43b..b81097c 100644 --- a/docs/best-practices/postgres/postgres-security-best-practices.md +++ b/docs/best-practices/postgres/postgres-security-best-practices.md @@ -1,6 +1,5 @@ # PostgreSQL Security Best Practices -**Objective**: Master senior-level PostgreSQL security patterns for production systems. When you need to implement robust security measures, when you want to protect sensitive data, when you need enterprise-grade security strategies—these best practices become your weapon of choice. ## Core Principles @@ -734,4 +733,3 @@ def setup_postgresql_security(): --- -*This guide provides the complete machinery for PostgreSQL security excellence. Each pattern includes implementation examples, security strategies, and real-world usage patterns for enterprise PostgreSQL security systems.* diff --git a/docs/best-practices/postgres/postgres-timeseries.md b/docs/best-practices/postgres/postgres-timeseries.md index 9150723..98ab4b5 100644 --- a/docs/best-practices/postgres/postgres-timeseries.md +++ b/docs/best-practices/postgres/postgres-timeseries.md @@ -1,6 +1,5 @@ # PostgreSQL Time Series Data Best Practices -**Objective**: Master senior-level PostgreSQL time series data patterns for production systems. When you need to handle time series data, when you want to optimize time-based queries, when you need enterprise-grade time series strategies—these best practices become your weapon of choice. ## Core Principles @@ -705,4 +704,3 @@ def setup_postgresql_timeseries(): --- -*This guide provides the complete machinery for PostgreSQL time series data excellence. Each pattern includes implementation examples, optimization strategies, and real-world usage patterns for enterprise PostgreSQL time series systems.* diff --git a/docs/best-practices/postgres/postgres-transactions-concurrency.md b/docs/best-practices/postgres/postgres-transactions-concurrency.md index f7a5883..65646fc 100644 --- a/docs/best-practices/postgres/postgres-transactions-concurrency.md +++ b/docs/best-practices/postgres/postgres-transactions-concurrency.md @@ -1,6 +1,5 @@ # PostgreSQL Transactions & Concurrency Best Practices -**Objective**: Master senior-level PostgreSQL transaction and concurrency patterns for production systems. When you need to handle concurrent access, when you want to optimize transaction performance, when you need enterprise-grade concurrency strategies—these best practices become your weapon of choice. ## Core Principles @@ -746,4 +745,3 @@ def setup_postgresql_transactions_concurrency(): --- -*This guide provides the complete machinery for PostgreSQL transactions and concurrency excellence. Each pattern includes implementation examples, concurrency strategies, and real-world usage patterns for enterprise PostgreSQL transaction systems.* diff --git a/docs/best-practices/postgres/postgres-troubleshooting.md b/docs/best-practices/postgres/postgres-troubleshooting.md index 614caee..169c472 100644 --- a/docs/best-practices/postgres/postgres-troubleshooting.md +++ b/docs/best-practices/postgres/postgres-troubleshooting.md @@ -1,6 +1,5 @@ # PostgreSQL Troubleshooting Best Practices -**Objective**: Master senior-level PostgreSQL troubleshooting patterns for production systems. When you need to diagnose database issues, when you want to implement systematic debugging, when you need enterprise-grade troubleshooting strategies—these best practices become your weapon of choice. ## Core Principles @@ -616,4 +615,3 @@ def setup_postgresql_troubleshooting(): --- -*This guide provides the complete machinery for PostgreSQL troubleshooting excellence. Each pattern includes implementation examples, diagnostic strategies, and real-world usage patterns for enterprise PostgreSQL troubleshooting systems.* diff --git a/docs/best-practices/python/dx-architecture-and-golden-paths.md b/docs/best-practices/python/dx-architecture-and-golden-paths.md index e025276..883e4e6 100644 --- a/docs/best-practices/python/dx-architecture-and-golden-paths.md +++ b/docs/best-practices/python/dx-architecture-and-golden-paths.md @@ -1,6 +1,5 @@ # Developer Experience (DX) as Infrastructure: Golden Paths, Tooling Ecosystems & Workflow Automation -**Objective**: Establish developer experience as infrastructure concern, providing golden paths, consistent tooling, and automated workflows that reduce cognitive load and operational entropy. When you need consistent scaffolding, when you want automated workflows, when you need AI-assisted development—this guide provides the complete framework. ## Introduction diff --git a/docs/best-practices/python/fastapi-geospatial.md b/docs/best-practices/python/fastapi-geospatial.md index 8282b75..4c01607 100644 --- a/docs/best-practices/python/fastapi-geospatial.md +++ b/docs/best-practices/python/fastapi-geospatial.md @@ -1,6 +1,5 @@ # FastAPI Geospatial API Development -**Objective**: Build high-performance, scalable geospatial APIs using FastAPI with proper spatial data handling and optimization. FastAPI provides the perfect foundation for geospatial APIs—async support, automatic OpenAPI documentation, and type safety. This guide covers building production-ready geospatial APIs that scale. diff --git a/docs/best-practices/python/index.md b/docs/best-practices/python/index.md index 0184fbe..2df77c4 100644 --- a/docs/best-practices/python/index.md +++ b/docs/best-practices/python/index.md @@ -1,6 +1,5 @@ # Python Development Best Practices -**Objective**: Master senior-level Python development patterns for production systems. When you need to build robust, scalable Python applications, when you want to follow proven methodologies, when you need enterprise-grade patterns—these best practices become your weapon of choice. ## Core Development @@ -63,4 +62,3 @@ --- -*These best practices provide the complete machinery for building production-ready Python systems. Each guide includes architectural patterns, configuration examples, and real-world implementation strategies for enterprise deployment.* diff --git a/docs/best-practices/python/pytest-best-practices.md b/docs/best-practices/python/pytest-best-practices.md index 949fcee..8ff7606 100644 --- a/docs/best-practices/python/pytest-best-practices.md +++ b/docs/best-practices/python/pytest-best-practices.md @@ -1,6 +1,5 @@ # Pytest Best Practices: Coverage, Plugins, Speed, CI -**Objective**: Master pytest for production-grade testing that's fast, reliable, and maintainable. When you need comprehensive test coverage, when you want to prevent regressions, when you're building scalable test suites—pytest becomes your weapon of choice. Pytest is the foundation of modern Python testing. Proper pytest configuration enables fast, reliable testing that prevents bugs and maintains code quality. This guide shows you how to wield pytest with the precision of a battle-tested backend engineer, covering everything from basic configuration to advanced CI integration. @@ -763,4 +762,3 @@ Pytest requires understanding both testing mechanics and CI integration patterns --- -*This guide provides the complete machinery for pytest testing. The patterns scale from simple unit tests to complex integration testing, from basic coverage to advanced CI automation.* diff --git a/docs/best-practices/python/python-api-design.md b/docs/best-practices/python/python-api-design.md index 5196ea6..1be787e 100644 --- a/docs/best-practices/python/python-api-design.md +++ b/docs/best-practices/python/python-api-design.md @@ -1,6 +1,5 @@ # Python API Design Best Practices -**Objective**: Master senior-level Python API design patterns for production systems. When you need to build robust, scalable APIs, when you want to implement comprehensive API versioning, when you need enterprise-grade API design strategies—these best practices become your weapon of choice. ## Core Principles @@ -1110,4 +1109,3 @@ def setup_api_design(): --- -*This guide provides the complete machinery for Python API design. Each pattern includes implementation examples, design strategies, and real-world usage patterns for enterprise API development.* diff --git a/docs/best-practices/python/python-async-best-practices.md b/docs/best-practices/python/python-async-best-practices.md index f2fe6d2..a78f650 100644 --- a/docs/best-practices/python/python-async-best-practices.md +++ b/docs/best-practices/python/python-async-best-practices.md @@ -1,6 +1,5 @@ # Python Async Best Practices: Write Coroutines That Don't Betray You -**Objective**: Master Python async programming with a focus on production reliability, performance, and maintainability. When you need to handle thousands of concurrent connections, when you want to keep UIs fluid, when you're building high-performance I/O systems—async becomes your weapon of choice. Async is a power tool. Use it well and you'll saturate sockets, keep UIs fluid, and squeeze latency out of I/O. Use it badly and you'll summon deadlocks, zombie tasks, and cancellation bugs that only appear at 3 a.m. This guide shows you how to wield async with the precision of a battle-tested backend engineer. @@ -1269,4 +1268,3 @@ Python async programming requires understanding cooperative scheduling, proper r --- -*This guide provides the complete machinery for mastering Python async programming. The patterns scale from simple I/O operations to complex distributed systems, from basic coroutines to advanced structured concurrency.* diff --git a/docs/best-practices/python/python-async-programming.md b/docs/best-practices/python/python-async-programming.md index c597c09..64c2bf9 100644 --- a/docs/best-practices/python/python-async-programming.md +++ b/docs/best-practices/python/python-async-programming.md @@ -1,6 +1,5 @@ # Python Async Programming Best Practices -**Objective**: Master senior-level Python async programming patterns for production systems. When you need to handle thousands of concurrent connections, when you want to build high-performance async applications, when you need enterprise-grade async programming strategies—these best practices become your weapon of choice. ## Core Principles @@ -934,4 +933,3 @@ async def setup_async_programming(): --- -*This guide provides the complete machinery for Python async programming. Each pattern includes implementation examples, error handling strategies, and real-world usage patterns for enterprise async development.* diff --git a/docs/best-practices/python/python-caching-strategies.md b/docs/best-practices/python/python-caching-strategies.md index 728e51c..3fac88e 100644 --- a/docs/best-practices/python/python-caching-strategies.md +++ b/docs/best-practices/python/python-caching-strategies.md @@ -1,6 +1,5 @@ # Python Caching Strategies Best Practices -**Objective**: Master senior-level Python caching patterns for production systems. When you need to implement high-performance caching, when you want to build distributed cache systems, when you need enterprise-grade caching strategies—these best practices become your weapon of choice. ## Core Principles @@ -1002,4 +1001,3 @@ def setup_caching_strategies(): --- -*This guide provides the complete machinery for Python caching strategies. Each pattern includes implementation examples, caching strategies, and real-world usage patterns for enterprise cache management.* diff --git a/docs/best-practices/python/python-cicd-pipelines.md b/docs/best-practices/python/python-cicd-pipelines.md index 0979f00..27c8d5e 100644 --- a/docs/best-practices/python/python-cicd-pipelines.md +++ b/docs/best-practices/python/python-cicd-pipelines.md @@ -1,6 +1,5 @@ # Python CI/CD Pipelines Best Practices -**Objective**: Master senior-level Python CI/CD pipeline patterns for production systems. When you need to build automated testing and deployment pipelines, when you want to implement comprehensive quality gates, when you need enterprise-grade CI/CD strategies—these best practices become your weapon of choice. ## Core Principles @@ -912,4 +911,3 @@ def setup_cicd_pipelines(): --- -*This guide provides the complete machinery for Python CI/CD pipelines. Each pattern includes implementation examples, automation strategies, and real-world usage patterns for enterprise pipeline management.* diff --git a/docs/best-practices/python/python-code-quality.md b/docs/best-practices/python/python-code-quality.md index 99b0e39..a69b76b 100644 --- a/docs/best-practices/python/python-code-quality.md +++ b/docs/best-practices/python/python-code-quality.md @@ -1,6 +1,5 @@ # Python Code Quality Best Practices -**Objective**: Master senior-level Python code quality patterns for production systems. When you need to maintain high code standards, when you want to implement comprehensive quality checks, when you need enterprise-grade code quality workflows—these best practices become your weapon of choice. ## Core Principles @@ -835,4 +834,3 @@ def setup_complete_code_quality(project_path: Path): --- -*This guide provides the complete machinery for Python code quality. Each pattern includes implementation examples, automation strategies, and real-world usage patterns for enterprise code quality management.* diff --git a/docs/best-practices/python/python-compliance.md b/docs/best-practices/python/python-compliance.md index 80a1255..e601163 100644 --- a/docs/best-practices/python/python-compliance.md +++ b/docs/best-practices/python/python-compliance.md @@ -1,6 +1,5 @@ # Python Compliance Best Practices -**Objective**: Master senior-level Python compliance patterns for production systems. When you need to implement regulatory compliance, when you want to build audit-ready applications, when you need enterprise-grade compliance strategies—these best practices become your weapon of choice. ## Core Principles @@ -1052,4 +1051,3 @@ def setup_compliance(): --- -*This guide provides the complete machinery for Python compliance best practices. Each pattern includes implementation examples, compliance strategies, and real-world usage patterns for enterprise compliance management.* diff --git a/docs/best-practices/python/python-concurrency-patterns.md b/docs/best-practices/python/python-concurrency-patterns.md index d101091..9674236 100644 --- a/docs/best-practices/python/python-concurrency-patterns.md +++ b/docs/best-practices/python/python-concurrency-patterns.md @@ -1,6 +1,5 @@ # Python Concurrency Patterns Best Practices -**Objective**: Master senior-level Python concurrency patterns for production systems. When you need to handle multiple tasks efficiently, when you want to leverage parallel processing, when you need enterprise-grade concurrency strategies—these best practices become your weapon of choice. ## Core Principles @@ -952,4 +951,3 @@ def setup_concurrency_patterns(): --- -*This guide provides the complete machinery for Python concurrency patterns. Each pattern includes implementation examples, synchronization strategies, and real-world usage patterns for enterprise concurrency management.* diff --git a/docs/best-practices/python/python-containerization.md b/docs/best-practices/python/python-containerization.md index efa3bc5..8258790 100644 --- a/docs/best-practices/python/python-containerization.md +++ b/docs/best-practices/python/python-containerization.md @@ -1,6 +1,5 @@ # Python Containerization Best Practices -**Objective**: Master senior-level Python containerization patterns for production systems. When you need to build efficient Docker images, when you want to implement multi-stage builds, when you need enterprise-grade container optimization—these best practices become your weapon of choice. ## Core Principles @@ -1160,4 +1159,3 @@ def setup_containerization(): --- -*This guide provides the complete machinery for Python containerization. Each pattern includes implementation examples, optimization strategies, and real-world usage patterns for enterprise container management.* diff --git a/docs/best-practices/python/python-data-analysis.md b/docs/best-practices/python/python-data-analysis.md index d57b9b0..e455c2c 100644 --- a/docs/best-practices/python/python-data-analysis.md +++ b/docs/best-practices/python/python-data-analysis.md @@ -1,6 +1,5 @@ # Python Data Analysis Best Practices -**Objective**: Master senior-level Python data analysis patterns for production systems. When you need to perform comprehensive data analysis, when you want to implement reproducible research workflows, when you need enterprise-grade data analysis strategies—these best practices become your weapon of choice. ## Core Principles @@ -908,4 +907,3 @@ def setup_data_analysis(): --- -*This guide provides the complete machinery for Python data analysis. Each pattern includes implementation examples, analysis strategies, and real-world usage patterns for enterprise data analysis management.* diff --git a/docs/best-practices/python/python-data-processing.md b/docs/best-practices/python/python-data-processing.md index 7bd2e41..d9331f1 100644 --- a/docs/best-practices/python/python-data-processing.md +++ b/docs/best-practices/python/python-data-processing.md @@ -1,6 +1,5 @@ # Python Data Processing Best Practices -**Objective**: Master senior-level Python data processing patterns for production systems. When you need to build scalable data pipelines, when you want to implement efficient ETL workflows, when you need enterprise-grade data processing strategies—these best practices become your weapon of choice. ## Core Principles @@ -1113,4 +1112,3 @@ def setup_data_processing(): --- -*This guide provides the complete machinery for Python data processing. Each pattern includes implementation examples, processing strategies, and real-world usage patterns for enterprise data processing management.* diff --git a/docs/best-practices/python/python-data-storage.md b/docs/best-practices/python/python-data-storage.md index 6bbe104..6614b18 100644 --- a/docs/best-practices/python/python-data-storage.md +++ b/docs/best-practices/python/python-data-storage.md @@ -1,6 +1,5 @@ # Python Data Storage Best Practices -**Objective**: Master senior-level Python data storage patterns for production systems. When you need to implement robust data persistence, when you want to build scalable storage solutions, when you need enterprise-grade data storage strategies—these best practices become your weapon of choice. ## Core Principles @@ -1087,4 +1086,3 @@ def setup_data_storage(): --- -*This guide provides the complete machinery for Python data storage. Each pattern includes implementation examples, storage strategies, and real-world usage patterns for enterprise data management.* diff --git a/docs/best-practices/python/python-database-patterns.md b/docs/best-practices/python/python-database-patterns.md index 39b53ad..f62e727 100644 --- a/docs/best-practices/python/python-database-patterns.md +++ b/docs/best-practices/python/python-database-patterns.md @@ -1,6 +1,5 @@ # Python Database Patterns Best Practices -**Objective**: Master senior-level Python database patterns for production systems. When you need to build robust database applications, when you want to implement efficient connection pooling, when you need enterprise-grade database strategies—these best practices become your weapon of choice. ## Core Principles @@ -982,4 +981,3 @@ def setup_database_patterns(): --- -*This guide provides the complete machinery for Python database patterns. Each pattern includes implementation examples, connection strategies, and real-world usage patterns for enterprise database management.* diff --git a/docs/best-practices/python/python-dev-environment.md b/docs/best-practices/python/python-dev-environment.md index d55a9fe..152a10b 100644 --- a/docs/best-practices/python/python-dev-environment.md +++ b/docs/best-practices/python/python-dev-environment.md @@ -1,6 +1,5 @@ # Python Development Environment Best Practices -**Objective**: Master senior-level Python development environment setup for production systems. When you need to manage Python versions, dependencies, and tooling efficiently, when you want to ensure reproducible development environments, when you need enterprise-grade development workflows—these best practices become your weapon of choice. ## Core Principles @@ -738,4 +737,3 @@ def create_python_development_environment(project_path: Path, project_name: str) --- -*This guide provides the complete machinery for setting up Python development environments. Each pattern includes implementation examples, configuration strategies, and real-world usage patterns for enterprise Python development.* diff --git a/docs/best-practices/python/python-error-handling.md b/docs/best-practices/python/python-error-handling.md index bdd262d..ba938e2 100644 --- a/docs/best-practices/python/python-error-handling.md +++ b/docs/best-practices/python/python-error-handling.md @@ -1,6 +1,5 @@ # Python Error Handling Best Practices -**Objective**: Master senior-level Python error handling patterns for production systems. When you need to build robust applications that handle failures gracefully, when you want to implement comprehensive error recovery, when you need enterprise-grade error handling strategies—these best practices become your weapon of choice. ## Core Principles @@ -902,4 +901,3 @@ def setup_error_handling(): --- -*This guide provides the complete machinery for Python error handling. Each pattern includes implementation examples, recovery strategies, and real-world usage patterns for enterprise error management.* diff --git a/docs/best-practices/python/python-geospatial-development.md b/docs/best-practices/python/python-geospatial-development.md index acd8d1d..adb1a7f 100644 --- a/docs/best-practices/python/python-geospatial-development.md +++ b/docs/best-practices/python/python-geospatial-development.md @@ -1,6 +1,5 @@ # Python Geospatial Development Best Practices -**Objective**: Master senior-level Python geospatial development patterns for production systems. When you need to build spatial data applications, when you want to implement geospatial analysis workflows, when you need enterprise-grade geospatial strategies—these best practices become your weapon of choice. ## Core Principles @@ -1048,4 +1047,3 @@ def setup_geospatial_development(): --- -*This guide provides the complete machinery for Python geospatial development. Each pattern includes implementation examples, spatial analysis strategies, and real-world usage patterns for enterprise geospatial management.* diff --git a/docs/best-practices/python/python-machine-learning.md b/docs/best-practices/python/python-machine-learning.md index 9ce2db5..206b9a2 100644 --- a/docs/best-practices/python/python-machine-learning.md +++ b/docs/best-practices/python/python-machine-learning.md @@ -1,6 +1,5 @@ # Python Machine Learning Best Practices -**Objective**: Master senior-level Python machine learning patterns for production systems. When you need to build robust ML pipelines, when you want to implement comprehensive model evaluation, when you need enterprise-grade ML strategies—these best practices become your weapon of choice. ## Core Principles @@ -956,4 +955,3 @@ def setup_ml_pipeline(): --- -*This guide provides the complete machinery for Python machine learning. Each pattern includes implementation examples, training strategies, and real-world usage patterns for enterprise ML management.* diff --git a/docs/best-practices/python/python-memory-management.md b/docs/best-practices/python/python-memory-management.md index 2ccb687..88f8cfb 100644 --- a/docs/best-practices/python/python-memory-management.md +++ b/docs/best-practices/python/python-memory-management.md @@ -1,6 +1,5 @@ # Python Memory Management Best Practices -**Objective**: Master senior-level Python memory management patterns for production systems. When you need to optimize memory usage, when you want to prevent memory leaks, when you need enterprise-grade memory management strategies—these best practices become your weapon of choice. ## Core Principles @@ -997,4 +996,3 @@ def setup_memory_management(): --- -*This guide provides the complete machinery for Python memory management. Each pattern includes implementation examples, optimization strategies, and real-world usage patterns for enterprise memory management.* diff --git a/docs/best-practices/python/python-microservices.md b/docs/best-practices/python/python-microservices.md index c749999..c74a860 100644 --- a/docs/best-practices/python/python-microservices.md +++ b/docs/best-practices/python/python-microservices.md @@ -1,6 +1,5 @@ # Python Microservices Best Practices -**Objective**: Master senior-level Python microservices patterns for production systems. When you need to build scalable, distributed applications, when you want to implement service communication patterns, when you need enterprise-grade microservices strategies—these best practices become your weapon of choice. ## Core Principles @@ -891,4 +890,3 @@ def setup_microservices(): --- -*This guide provides the complete machinery for Python microservices. Each pattern includes implementation examples, communication strategies, and real-world usage patterns for enterprise microservices development.* diff --git a/docs/best-practices/python/python-monitoring-observability.md b/docs/best-practices/python/python-monitoring-observability.md index 18cba46..c6594b7 100644 --- a/docs/best-practices/python/python-monitoring-observability.md +++ b/docs/best-practices/python/python-monitoring-observability.md @@ -1,6 +1,5 @@ # Python Monitoring & Observability Best Practices -**Objective**: Master senior-level Python monitoring and observability patterns for production systems. When you need to implement comprehensive monitoring, when you want to build distributed tracing, when you need enterprise-grade observability strategies—these best practices become your weapon of choice. ## Core Principles @@ -1102,4 +1101,3 @@ def setup_monitoring_observability(): --- -*This guide provides the complete machinery for Python monitoring and observability. Each pattern includes implementation examples, monitoring strategies, and real-world usage patterns for enterprise observability management.* diff --git a/docs/best-practices/python/python-package-development.md b/docs/best-practices/python/python-package-development.md index 28350a4..f7e7fc7 100644 --- a/docs/best-practices/python/python-package-development.md +++ b/docs/best-practices/python/python-package-development.md @@ -1,6 +1,5 @@ # Python Package Development Best Practices -**Objective**: Master senior-level Python package development patterns for production systems. When you need to build and ship Python packages, when you want to ensure reproducibility and maintainability, when you need enterprise-grade packaging workflows—these best practices become your weapon of choice. ## Core Principles @@ -1024,4 +1023,3 @@ def create_complete_python_package(package_path: Path, package_name: str): --- -*This guide provides the complete machinery for Python package development. Each pattern includes implementation examples, configuration strategies, and real-world usage patterns for enterprise Python packaging.* diff --git a/docs/best-practices/python/python-performance-tuning.md b/docs/best-practices/python/python-performance-tuning.md index 566a644..aa6e05d 100644 --- a/docs/best-practices/python/python-performance-tuning.md +++ b/docs/best-practices/python/python-performance-tuning.md @@ -1,6 +1,5 @@ # Python Performance Tuning Best Practices -**Objective**: Master senior-level Python performance tuning patterns for production systems. When you need to optimize Python applications for speed and efficiency, when you want to identify and resolve performance bottlenecks, when you need enterprise-grade performance optimization strategies—these best practices become your weapon of choice. ## Core Principles @@ -737,4 +736,3 @@ def optimize_python_application(): --- -*This guide provides the complete machinery for Python performance tuning. Each pattern includes implementation examples, optimization strategies, and real-world usage patterns for enterprise performance optimization.* diff --git a/docs/best-practices/python/python-production-deployment.md b/docs/best-practices/python/python-production-deployment.md index d2be162..2d90a73 100644 --- a/docs/best-practices/python/python-production-deployment.md +++ b/docs/best-practices/python/python-production-deployment.md @@ -1,6 +1,5 @@ # Python Production Deployment Best Practices -**Objective**: Master senior-level Python production deployment patterns for enterprise systems. When you need to deploy applications at scale, when you want to implement zero-downtime deployments, when you need enterprise-grade deployment strategies—these best practices become your weapon of choice. ## Core Principles @@ -1149,4 +1148,3 @@ def setup_production_deployment(): --- -*This guide provides the complete machinery for Python production deployment. Each pattern includes implementation examples, deployment strategies, and real-world usage patterns for enterprise deployment management.* diff --git a/docs/best-practices/python/python-secrets-management.md b/docs/best-practices/python/python-secrets-management.md index 5f88b58..d991c2a 100644 --- a/docs/best-practices/python/python-secrets-management.md +++ b/docs/best-practices/python/python-secrets-management.md @@ -1,6 +1,5 @@ # Python Secrets Management Best Practices -**Objective**: Master senior-level Python secrets management patterns for production systems. When you need to implement secure credential handling, when you want to build robust secret rotation, when you need enterprise-grade secrets management—these best practices become your weapon of choice. ## Core Principles @@ -1032,4 +1031,3 @@ def setup_secrets_management(): --- -*This guide provides the complete machinery for Python secrets management. Each pattern includes implementation examples, rotation strategies, and real-world usage patterns for enterprise secrets management.* diff --git a/docs/best-practices/python/python-security-best-practices.md b/docs/best-practices/python/python-security-best-practices.md index 6e640c0..1ebc3f1 100644 --- a/docs/best-practices/python/python-security-best-practices.md +++ b/docs/best-practices/python/python-security-best-practices.md @@ -1,6 +1,5 @@ # Python Security Best Practices -**Objective**: Master senior-level Python security patterns for production systems. When you need to implement comprehensive security measures, when you want to build secure applications, when you need enterprise-grade security strategies—these best practices become your weapon of choice. ## Core Principles @@ -1051,4 +1050,3 @@ def setup_security(): --- -*This guide provides the complete machinery for Python security best practices. Each pattern includes implementation examples, security strategies, and real-world usage patterns for enterprise security management.* diff --git a/docs/best-practices/python/python-testing-best-practices.md b/docs/best-practices/python/python-testing-best-practices.md index 9f3bd0c..ccf6a89 100644 --- a/docs/best-practices/python/python-testing-best-practices.md +++ b/docs/best-practices/python/python-testing-best-practices.md @@ -1,6 +1,5 @@ # Python Testing Best Practices -**Objective**: Master senior-level Python testing patterns for production systems. When you need to ensure code reliability and maintainability, when you want to implement comprehensive testing strategies, when you need enterprise-grade testing workflows—these best practices become your weapon of choice. ## Core Principles @@ -811,4 +810,3 @@ def setup_comprehensive_testing(project_path: Path): --- -*This guide provides the complete machinery for Python testing. Each pattern includes implementation examples, testing strategies, and real-world usage patterns for enterprise Python testing.* diff --git a/docs/best-practices/python/python-threading-and-multiprocessing.md b/docs/best-practices/python/python-threading-and-multiprocessing.md index 5a22a9d..c878af7 100644 --- a/docs/best-practices/python/python-threading-and-multiprocessing.md +++ b/docs/best-practices/python/python-threading-and-multiprocessing.md @@ -1,6 +1,5 @@ # Python Concurrency: Threads vs Processes (and Doing Pools the Right Way) -**Objective**: Master Python concurrency with a focus on production performance, safety, and scalability. When you need to handle I/O-bound workloads, when you want true CPU parallelism, when you're building production systems—concurrency becomes your weapon of choice. The GIL is not your enemy, but it's not your friend either. Understanding when to use threads, processes, or asyncio determines whether your application scales or crumbles under load. This guide shows you how to wield Python concurrency with the precision of a battle-tested backend engineer. @@ -1088,4 +1087,3 @@ Python concurrency requires understanding the GIL, choosing the right tool for t --- -*This guide provides the complete machinery for mastering Python concurrency. The patterns scale from simple I/O operations to complex distributed processing, from basic threading to advanced multiprocessing architectures.* diff --git a/docs/best-practices/python/python-type-hints.md b/docs/best-practices/python/python-type-hints.md index e69b952..67aae76 100644 --- a/docs/best-practices/python/python-type-hints.md +++ b/docs/best-practices/python/python-type-hints.md @@ -1,6 +1,5 @@ # Python Type Hints Best Practices -**Objective**: Master senior-level Python type hints patterns for production systems. When you need to build type-safe Python applications, when you want to leverage static analysis tools, when you need enterprise-grade type safety strategies—these best practices become your weapon of choice. ## Core Principles @@ -979,4 +978,3 @@ def setup_type_hints(): --- -*This guide provides the complete machinery for Python type hints. Each pattern includes implementation examples, type safety strategies, and real-world usage patterns for enterprise type-safe development.* diff --git a/docs/best-practices/python/python-web-performance.md b/docs/best-practices/python/python-web-performance.md index 8de5574..06a7443 100644 --- a/docs/best-practices/python/python-web-performance.md +++ b/docs/best-practices/python/python-web-performance.md @@ -1,6 +1,5 @@ # Python Web Performance Best Practices -**Objective**: Master senior-level Python web performance patterns for production systems. When you need to optimize web applications for speed and scalability, when you want to implement comprehensive performance monitoring, when you need enterprise-grade web performance strategies—these best practices become your weapon of choice. ## Core Principles @@ -997,4 +996,3 @@ def setup_web_performance(): --- -*This guide provides the complete machinery for Python web performance. Each pattern includes implementation examples, optimization strategies, and real-world usage patterns for enterprise web performance management.* diff --git a/docs/best-practices/python/python-web-services.md b/docs/best-practices/python/python-web-services.md index 86f72ec..9974b1f 100644 --- a/docs/best-practices/python/python-web-services.md +++ b/docs/best-practices/python/python-web-services.md @@ -1,6 +1,5 @@ # Python Web Services Best Practices -**Objective**: Master senior-level Python web service patterns for production systems. When you need to build scalable, reliable web services, when you want to implement modern web frameworks, when you need enterprise-grade web service architectures—these best practices become your weapon of choice. ## Core Principles @@ -836,4 +835,3 @@ def create_production_web_service(framework: str = "fastapi"): --- -*This guide provides the complete machinery for Python web services. Each pattern includes implementation examples, framework-specific strategies, and real-world usage patterns for enterprise web service development.* diff --git a/docs/best-practices/python/typing-in-python.md b/docs/best-practices/python/typing-in-python.md index 929e8c7..3d0a6b0 100644 --- a/docs/best-practices/python/typing-in-python.md +++ b/docs/best-practices/python/typing-in-python.md @@ -1,1004 +1,3 @@ -# Typing in Python: The Art of Type Safety +# Typing in Python -**Objective**: Master Python's type system to write bulletproof code that catches errors before they reach production. When your codebase grows beyond comprehension, when runtime errors plague your deployments, when refactoring becomes a nightmare—type hints become your weapon of choice. - -Python's type system is the bridge between dynamic flexibility and static safety. Without proper typing, you're flying blind into production with code that could fail in ways you never imagined. This guide shows you how to wield Python's type system with the precision of a seasoned engineer. - -## 0) Prerequisites (Read Once, Live by Them) - -### The Five Commandments - -1. **Understand the type system** - - Static vs dynamic typing - - Type hints vs runtime behavior - - Gradual typing philosophy - -2. **Master the core types** - - Built-in types and collections - - Generic types and type variables - - Union types and optional values - -3. **Know your tools** - - mypy for static analysis - - pyright for VS Code - - pyre for Facebook's type checker - -4. **Validate everything** - - Type checking in CI/CD - - Runtime type validation - - Performance implications - -5. **Plan for production** - - Gradual adoption strategy - - Legacy code integration - - Team collaboration - -**Why These Principles**: Type hints are the foundation of reliable Python code. Understanding the type system, mastering the tools, and following best practices is essential for maintaining code quality at scale. - -## 1) The Foundation (Core Types) - -### Basic Type Hints - -```python -# Basic types -def greet(name: str) -> str: - return f"Hello, {name}!" - -def calculate_area(length: float, width: float) -> float: - return length * width - -def is_even(number: int) -> bool: - return number % 2 == 0 - -# None type -def find_user(user_id: int) -> str | None: - # Returns None if user not found - return None - -# Optional (legacy, use | None instead) -from typing import Optional -def find_user_legacy(user_id: int) -> Optional[str]: - return None -``` - -### Collection Types - -```python -from typing import List, Dict, Set, Tuple, FrozenSet - -# Lists -def process_items(items: List[str]) -> List[int]: - return [len(item) for item in items] - -# Dictionaries -def create_user_map(users: List[str]) -> Dict[str, int]: - return {user: len(user) for user in users} - -# Sets -def unique_items(items: List[str]) -> Set[str]: - return set(items) - -# Tuples -def get_coordinates() -> Tuple[float, float]: - return (40.7128, -74.0060) - -# Frozen sets -def immutable_items(items: List[str]) -> FrozenSet[str]: - return frozenset(items) -``` - -### Modern Collection Types (Python 3.9+) - -```python -# Use built-in types instead of typing module -def process_items(items: list[str]) -> list[int]: - return [len(item) for item in items] - -def create_user_map(users: list[str]) -> dict[str, int]: - return {user: len(user) for user in users} - -def unique_items(items: list[str]) -> set[str]: - return set(items) - -def get_coordinates() -> tuple[float, float]: - return (40.7128, -74.0060) -``` - -**Why These Types**: Basic type hints provide immediate benefits for function signatures. They catch type errors at development time and serve as documentation for other developers. - -## 2) Advanced Types (The Power) - -### Generic Types - -```python -from typing import TypeVar, Generic, Protocol, runtime_checkable - -# Type variables -T = TypeVar('T') -K = TypeVar('K') -V = TypeVar('V') - -class Stack(Generic[T]): - def __init__(self) -> None: - self._items: list[T] = [] - - def push(self, item: T) -> None: - self._items.append(item) - - def pop(self) -> T: - return self._items.pop() - - def is_empty(self) -> bool: - return len(self._items) == 0 - -# Usage -string_stack: Stack[str] = Stack() -string_stack.push("hello") -string_stack.push("world") - -number_stack: Stack[int] = Stack() -number_stack.push(42) -number_stack.push(13) -``` - -### Protocol Types (Structural Typing) - -```python -from typing import Protocol, runtime_checkable - -@runtime_checkable -class Drawable(Protocol): - def draw(self) -> None: ... - def get_area(self) -> float: ... - -class Circle: - def __init__(self, radius: float) -> None: - self.radius = radius - - def draw(self) -> None: - print(f"Drawing circle with radius {self.radius}") - - def get_area(self) -> float: - return 3.14159 * self.radius ** 2 - -class Rectangle: - def __init__(self, width: float, height: float) -> None: - self.width = width - self.height = height - - def draw(self) -> None: - print(f"Drawing rectangle {self.width}x{self.height}") - - def get_area(self) -> float: - return self.width * self.height - -def render_shape(shape: Drawable) -> None: - shape.draw() - print(f"Area: {shape.get_area()}") - -# Usage -circle = Circle(5.0) -rectangle = Rectangle(10.0, 20.0) - -render_shape(circle) # Works -render_shape(rectangle) # Works -``` - -### Union Types and Literals - -```python -from typing import Union, Literal, Final - -# Union types (use | in Python 3.10+) -def process_id(user_id: int | str) -> str: - return str(user_id) - -# Legacy union syntax -def process_id_legacy(user_id: Union[int, str]) -> str: - return str(user_id) - -# Literal types -def get_status() -> Literal["success", "error", "pending"]: - return "success" - -# Final constants -API_VERSION: Final = "v1" -MAX_RETRIES: Final = 3 - -# Typed dictionaries -from typing import TypedDict - -class UserDict(TypedDict): - id: int - name: str - email: str - is_active: bool - -def create_user(user_data: UserDict) -> UserDict: - return user_data -``` - -### Callable Types - -```python -from typing import Callable, Awaitable - -# Function types -def apply_function(func: Callable[[int, int], int], a: int, b: int) -> int: - return func(a, b) - -# Async function types -async def apply_async_function(func: Callable[[int, int], Awaitable[int]], a: int, b: int) -> int: - return await func(a, b) - -# Higher-order functions -def create_multiplier(factor: int) -> Callable[[int], int]: - def multiply(x: int) -> int: - return x * factor - return multiply - -# Usage -multiply_by_3 = create_multiplier(3) -result = multiply_by_3(5) # Returns 15 -``` - -**Why Advanced Types**: Generic types enable reusable, type-safe code. Protocols provide structural typing without inheritance. Union types handle multiple possible types safely. - -## 3) Type Checking Tools (The Arsenal) - -### mypy Configuration - -```ini -# mypy.ini -[mypy] -python_version = 3.11 -warn_return_any = True -warn_unused_configs = True -disallow_untyped_defs = True -disallow_incomplete_defs = True -check_untyped_defs = True -disallow_untyped_decorators = True -no_implicit_optional = True -warn_redundant_casts = True -warn_unused_ignores = True -warn_no_return = True -warn_unreachable = True -strict_optional = True -show_error_codes = True - -# Per-module options -[mypy-tests.*] -disallow_untyped_defs = False - -[mypy-legacy.*] -ignore_errors = True -``` - -### pyright Configuration - -```json -{ - "include": ["src"], - "exclude": ["**/node_modules", "**/__pycache__"], - "reportMissingImports": true, - "reportMissingTypeStubs": false, - "pythonVersion": "3.11", - "pythonPlatform": "Linux", - "executionEnvironments": [ - { - "root": "src", - "pythonVersion": "3.11" - } - ], - "typeCheckingMode": "strict", - "useLibraryCodeForTypes": true, - "autoImportCompletions": true, - "strictListInference": true, - "strictDictionaryInference": true, - "strictSetInference": true -} -``` - -### CI/CD Integration - -```yaml -# .github/workflows/type-check.yml -name: Type Checking - -on: - push: - branches: [main, develop] - pull_request: - branches: [main] - -jobs: - type-check: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: [3.9, 3.10, 3.11] - - steps: - - uses: actions/checkout@v3 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - pip install -r requirements.txt - pip install mypy pyright - - - name: Run mypy - run: mypy src/ - - - name: Run pyright - run: pyright src/ -``` - -**Why These Tools**: Type checkers catch errors before runtime. Proper configuration ensures consistent type checking across the team. CI/CD integration prevents type errors from reaching production. - -## 4) Runtime Type Validation (The Safety Net) - -### Pydantic Integration - -```python -from pydantic import BaseModel, Field, validator -from typing import Optional, List -from datetime import datetime - -class User(BaseModel): - id: int - name: str = Field(..., min_length=1, max_length=100) - email: str = Field(..., regex=r'^[^@]+@[^@]+\.[^@]+$') - age: Optional[int] = Field(None, ge=0, le=150) - tags: List[str] = Field(default_factory=list) - created_at: datetime = Field(default_factory=datetime.utcnow) - - @validator('name') - def name_must_not_be_empty(cls, v): - if not v.strip(): - raise ValueError('Name cannot be empty') - return v.strip() - - @validator('tags') - def tags_must_be_unique(cls, v): - if len(v) != len(set(v)): - raise ValueError('Tags must be unique') - return v - -# Usage -try: - user = User( - id=1, - name="John Doe", - email="john@example.com", - age=30, - tags=["developer", "python"] - ) - print(user.json()) -except ValueError as e: - print(f"Validation error: {e}") -``` - -### Custom Type Validators - -```python -from typing import Any, Type, Union -import re - -def validate_email(value: str) -> str: - """Validate email format""" - pattern = r'^[^@]+@[^@]+\.[^@]+$' - if not re.match(pattern, value): - raise ValueError(f"Invalid email format: {value}") - return value - -def validate_positive_int(value: int) -> int: - """Validate positive integer""" - if value <= 0: - raise ValueError(f"Value must be positive: {value}") - return value - -def validate_phone_number(value: str) -> str: - """Validate phone number format""" - pattern = r'^\+?1?[-.\s]?\(?([0-9]{3})\)?[-.\s]?([0-9]{3})[-.\s]?([0-9]{4})$' - if not re.match(pattern, value): - raise ValueError(f"Invalid phone number format: {value}") - return value - -# Usage in functions -def create_user(name: str, email: str, phone: str) -> dict[str, Any]: - validated_email = validate_email(email) - validated_phone = validate_phone_number(phone) - - return { - "name": name, - "email": validated_email, - "phone": validated_phone - } -``` - -### Type Guards - -```python -from typing import TypeGuard, Union - -def is_string(value: Union[str, int]) -> TypeGuard[str]: - """Type guard to check if value is a string""" - return isinstance(value, str) - -def is_positive_number(value: Union[int, float, str]) -> TypeGuard[Union[int, float]]: - """Type guard to check if value is a positive number""" - if isinstance(value, (int, float)): - return value > 0 - return False - -def process_value(value: Union[str, int, float]) -> str: - if is_string(value): - # mypy knows value is str here - return f"String: {value.upper()}" - elif is_positive_number(value): - # mypy knows value is int | float here - return f"Positive number: {value * 2}" - else: - return f"Other: {value}" -``` - -**Why Runtime Validation**: Type hints are erased at runtime. Runtime validation ensures data integrity and provides meaningful error messages. - -## 5) Advanced Patterns (The Art) - -### Generic Protocols - -```python -from typing import Protocol, TypeVar, Generic - -T = TypeVar('T') - -class Comparable(Protocol): - def __lt__(self, other: 'Comparable') -> bool: ... - def __le__(self, other: 'Comparable') -> bool: ... - def __gt__(self, other: 'Comparable') -> bool: ... - def __ge__(self, other: 'Comparable') -> bool: ... - -def find_max(items: list[T]) -> T: - """Find maximum item in a list""" - if not items: - raise ValueError("Cannot find max of empty list") - - max_item = items[0] - for item in items[1:]: - if item > max_item: # Type checker knows T is Comparable - max_item = item - return max_item - -# Usage -numbers = [1, 5, 3, 9, 2] -max_number = find_max(numbers) # Returns 9 - -strings = ["apple", "banana", "cherry"] -max_string = find_max(strings) # Returns "cherry" -``` - -### Overloaded Functions - -```python -from typing import overload, Union - -@overload -def process_data(data: str) -> str: ... - -@overload -def process_data(data: int) -> int: ... - -@overload -def process_data(data: list[str]) -> list[str]: ... - -def process_data(data: Union[str, int, list[str]]) -> Union[str, int, list[str]]: - """Process data based on its type""" - if isinstance(data, str): - return data.upper() - elif isinstance(data, int): - return data * 2 - elif isinstance(data, list): - return [item.upper() for item in data] - else: - raise TypeError(f"Unsupported type: {type(data)}") - -# Usage -result1 = process_data("hello") # Returns "HELLO" -result2 = process_data(42) # Returns 84 -result3 = process_data(["a", "b"]) # Returns ["A", "B"] -``` - -### Generic Classes with Constraints - -```python -from typing import TypeVar, Generic, Protocol - -class Addable(Protocol): - def __add__(self, other: 'Addable') -> 'Addable': ... - -T = TypeVar('T', bound=Addable) - -class Calculator(Generic[T]): - def __init__(self, initial_value: T) -> None: - self.value = initial_value - - def add(self, other: T) -> 'Calculator[T]': - self.value = self.value + other - return self - - def get_value(self) -> T: - return self.value - -# Usage -int_calc = Calculator(10) -int_calc.add(5).add(3) -print(int_calc.get_value()) # Returns 18 - -str_calc = Calculator("Hello") -str_calc.add(" ").add("World") -print(str_calc.get_value()) # Returns "Hello World" -``` - -### Typed Decorators - -```python -from typing import Callable, TypeVar, ParamSpec, Any -from functools import wraps -import time - -P = ParamSpec('P') -T = TypeVar('T') - -def timing(func: Callable[P, T]) -> Callable[P, T]: - """Decorator to measure function execution time""" - @wraps(func) - def wrapper(*args: P.args, **kwargs: P.kwargs) -> T: - start = time.time() - result = func(*args, **kwargs) - end = time.time() - print(f"{func.__name__} took {end - start:.4f} seconds") - return result - return wrapper - -def retry(max_attempts: int = 3): - """Decorator to retry function on failure""" - def decorator(func: Callable[P, T]) -> Callable[P, T]: - @wraps(func) - def wrapper(*args: P.args, **kwargs: P.kwargs) -> T: - last_exception = None - for attempt in range(max_attempts): - try: - return func(*args, **kwargs) - except Exception as e: - last_exception = e - if attempt == max_attempts - 1: - raise - raise last_exception - return wrapper - return decorator - -# Usage -@timing -@retry(max_attempts=3) -def risky_operation(x: int) -> int: - if x < 0: - raise ValueError("Negative values not allowed") - return x * 2 -``` - -**Why Advanced Patterns**: These patterns enable sophisticated type-safe code. They provide compile-time safety while maintaining runtime flexibility. - -## 6) Performance Considerations (The Reality) - -### Type Checking Performance - -```python -# Avoid expensive type checks in hot paths -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - # These imports are only used for type checking - from expensive_module import HeavyClass - from another_module import ComplexType - -def process_data(data: 'HeavyClass') -> 'ComplexType': - # Function implementation - pass - -# Use string literals for forward references -class Node: - def __init__(self, value: int) -> None: - self.value = value - self.children: list['Node'] = [] - - def add_child(self, child: 'Node') -> None: - self.children.append(child) -``` - -### Runtime Type Checking - -```python -from typing import get_type_hints, get_origin, get_args -import inspect - -def validate_function_args(func: Callable, args: tuple, kwargs: dict) -> None: - """Validate function arguments against type hints""" - hints = get_type_hints(func) - - # Get parameter names - sig = inspect.signature(func) - param_names = list(sig.parameters.keys()) - - # Check positional arguments - for i, (name, hint) in enumerate(zip(param_names, hints.values())): - if i < len(args): - value = args[i] - if not isinstance(value, hint): - raise TypeError(f"Argument {name} expected {hint}, got {type(value)}") - - # Check keyword arguments - for name, value in kwargs.items(): - if name in hints: - hint = hints[name] - if not isinstance(value, hint): - raise TypeError(f"Argument {name} expected {hint}, got {type(value)}") - -# Usage -def add_numbers(a: int, b: int) -> int: - return a + b - -# This will raise TypeError -try: - validate_function_args(add_numbers, (1, "2"), {}) -except TypeError as e: - print(f"Type error: {e}") -``` - -### Memory Optimization - -```python -from typing import Final, ClassVar -from dataclasses import dataclass - -# Use Final for constants -API_VERSION: Final = "v1" -MAX_CONNECTIONS: Final = 100 - -# Use ClassVar for class-level constants -class DatabaseConfig: - HOST: ClassVar[str] = "localhost" - PORT: ClassVar[int] = 5432 - TIMEOUT: ClassVar[float] = 30.0 - -# Use dataclasses for memory efficiency -@dataclass(frozen=True) -class Point: - x: float - y: float - - def distance_to(self, other: 'Point') -> float: - return ((self.x - other.x) ** 2 + (self.y - other.y) ** 2) ** 0.5 - -# Use __slots__ for memory efficiency -class User: - __slots__ = ['id', 'name', 'email'] - - def __init__(self, id: int, name: str, email: str) -> None: - self.id = id - self.name = name - self.email = email -``` - -**Why Performance Matters**: Type checking adds overhead. Understanding the performance implications helps you make informed decisions about where to apply strict typing. - -## 7) Best Practices (The Wisdom) - -### Gradual Adoption Strategy - -```python -# Start with new code -def new_function(data: list[str]) -> dict[str, int]: - return {item: len(item) for item in data} - -# Gradually add types to existing code -def existing_function(data): # No types yet - return [len(item) for item in data] - -# Add types incrementally -def existing_function_typed(data: list[str]) -> list[int]: - return [len(item) for item in data] - -# Use type: ignore for legacy code -def legacy_function(data): # type: ignore - return [len(item) for item in data] -``` - -### Team Collaboration - -```python -# Use consistent type hints across the team -from typing import NewType, NamedTuple - -# Create domain-specific types -UserId = NewType('UserId', int) -Email = NewType('Email', str) - -class UserInfo(NamedTuple): - id: UserId - name: str - email: Email - is_active: bool - -def create_user(user_id: UserId, name: str, email: Email) -> UserInfo: - return UserInfo( - id=user_id, - name=name, - email=email, - is_active=True - ) - -# Usage -user_id = UserId(123) -email = Email("user@example.com") -user = create_user(user_id, "John Doe", email) -``` - -### Error Handling with Types - -```python -from typing import Union, Optional -from dataclasses import dataclass - -@dataclass -class ValidationError: - field: str - message: str - value: Any - -@dataclass -class Success: - value: Any - -Result = Union[Success, ValidationError] - -def validate_user(name: str, email: str, age: int) -> Result: - if not name.strip(): - return ValidationError("name", "Name cannot be empty", name) - - if "@" not in email: - return ValidationError("email", "Invalid email format", email) - - if age < 0: - return ValidationError("age", "Age cannot be negative", age) - - return Success({"name": name, "email": email, "age": age}) - -# Usage -result = validate_user("John", "john@example.com", 30) -if isinstance(result, Success): - print(f"User created: {result.value}") -else: - print(f"Validation error: {result.field} - {result.message}") -``` - -### Testing with Types - -```python -from typing import TypeVar, Generic -import pytest - -T = TypeVar('T') - -class TestStack(Generic[T]): - def __init__(self) -> None: - self._items: list[T] = [] - - def push(self, item: T) -> None: - self._items.append(item) - - def pop(self) -> T: - return self._items.pop() - - def is_empty(self) -> bool: - return len(self._items) == 0 - -# Type-safe test cases -def test_stack_operations() -> None: - stack: TestStack[int] = TestStack() - - assert stack.is_empty() - - stack.push(1) - stack.push(2) - - assert not stack.is_empty() - assert stack.pop() == 2 - assert stack.pop() == 1 - assert stack.is_empty() - -# Run with mypy -# mypy test_stack.py -``` - -**Why These Practices**: Gradual adoption enables teams to benefit from typing without rewriting everything. Consistent practices ensure maintainable code across the team. - -## 8) Common Pitfalls (The Traps) - -### Type Erasure - -```python -# Types are erased at runtime -def process_data(data: list[str]) -> list[int]: - return [len(item) for item in data] - -# This will NOT raise a TypeError at runtime -result = process_data([1, 2, 3]) # Wrong type, but no runtime error - -# Use runtime validation for critical paths -def safe_process_data(data: list[str]) -> list[int]: - if not all(isinstance(item, str) for item in data): - raise TypeError("All items must be strings") - return [len(item) for item in data] -``` - -### Circular Imports - -```python -# Avoid circular imports with TYPE_CHECKING -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from .user import User - from .order import Order - -class Customer: - def __init__(self, name: str) -> None: - self.name = name - self.orders: list['Order'] = [] - - def add_order(self, order: 'Order') -> None: - self.orders.append(order) - - def get_user(self) -> 'User': - # Implementation - pass -``` - -### Generic Type Confusion - -```python -# Avoid this common mistake -from typing import List, TypeVar - -T = TypeVar('T') - -# Wrong: This doesn't work as expected -def process_items(items: List[T]) -> List[T]: - return items # Type checker doesn't know what T is - -# Correct: Use constraints or protocols -class Comparable(Protocol): - def __lt__(self, other: 'Comparable') -> bool: ... - -def find_max(items: list[T]) -> T: - if not items: - raise ValueError("Empty list") - return max(items) # This works because max() handles the comparison -``` - -### Over-typing - -```python -# Don't over-type simple cases -def add_numbers(a: int, b: int) -> int: - return a + b - -# This is overkill for simple functions -def simple_operation(x: int) -> int: - return x * 2 - -# Focus on complex functions and public APIs -def complex_data_processing( - data: list[dict[str, Union[str, int, float]]], - filters: dict[str, Any], - transformers: list[Callable[[Any], Any]] -) -> list[dict[str, Any]]: - # Complex logic here - pass -``` - -**Why These Pitfalls Matter**: Understanding common mistakes prevents type system abuse. These examples show how to avoid common typing pitfalls. - -## 9) TL;DR Quickstart (The Essentials) - -### Essential Commands - -```bash -# Install type checking tools -pip install mypy pyright - -# Run type checking -mypy src/ -pyright src/ - -# Configure mypy -echo "[mypy] -python_version = 3.11 -strict = True" > mypy.ini - -# Add to pre-commit -pip install pre-commit -echo "repos: -- repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.0.1 - hooks: - - id: mypy" > .pre-commit-config.yaml -``` - -### Essential Type Hints - -```python -# Basic types -def greet(name: str) -> str: - return f"Hello, {name}!" - -# Collections -def process_items(items: list[str]) -> dict[str, int]: - return {item: len(item) for item in items} - -# Optional values -def find_user(user_id: int) -> str | None: - return None - -# Generic types -T = TypeVar('T') -def identity(value: T) -> T: - return value - -# Protocols -class Drawable(Protocol): - def draw(self) -> None: ... - -def render(shape: Drawable) -> None: - shape.draw() -``` - -### Essential Configuration - -```python -# mypy.ini -[mypy] -python_version = 3.11 -strict = True -warn_return_any = True -disallow_untyped_defs = True - -# pyrightconfig.json -{ - "include": ["src"], - "exclude": ["**/node_modules"], - "typeCheckingMode": "strict" -} -``` - -**Why This Quickstart**: These commands and patterns cover 90% of daily typing usage. Master these before exploring advanced features. - -## 10) The Machine's Summary - -Python's type system is the foundation of reliable code. When configured properly, it provides compile-time safety, runtime validation, and clear documentation. The key is understanding the type system, mastering the tools, and following best practices. - -**The Dark Truth**: Without type hints, your Python code is a ticking time bomb. Type hints are your safety net. Use them wisely. - -**The Machine's Mantra**: "In types we trust, in static analysis we build, and in the code we find the path to reliability." - -**Why This Matters**: Type hints enable confident refactoring, catch errors early, and serve as living documentation. They transform Python from a dynamic language into a statically-typed powerhouse. - ---- - -*This tutorial provides the complete machinery for mastering Python's type system. The patterns scale from development to production, from simple functions to complex generic algorithms.* +This page was consolidated into **[Python type hints](python-type-hints.md)**. diff --git a/docs/best-practices/python/web-performance-optimization.md b/docs/best-practices/python/web-performance-optimization.md index acf131b..5ecfad3 100644 --- a/docs/best-practices/python/web-performance-optimization.md +++ b/docs/best-practices/python/web-performance-optimization.md @@ -1,6 +1,5 @@ # Web Performance Optimization -**Objective**: Achieve optimal web performance through Core Web Vitals optimization, modern development practices, and comprehensive monitoring. Web performance directly impacts user experience, SEO rankings, and business metrics. This guide covers achieving sub-2.5s LCP, <100ms FID, and <0.1 CLS through proven optimization techniques. diff --git a/docs/best-practices/r/index.md b/docs/best-practices/r/index.md index 33674cb..5228958 100644 --- a/docs/best-practices/r/index.md +++ b/docs/best-practices/r/index.md @@ -1,6 +1,5 @@ # R Development Best Practices -**Objective**: Master senior-level R development patterns for production systems. When you need to build robust, scalable R applications, when you want to follow proven methodologies, when you need enterprise-grade patterns—these best practices become your weapon of choice. ## Core R Development @@ -42,4 +41,3 @@ --- -*These best practices provide the complete machinery for building production-ready R systems. Each guide includes architectural patterns, configuration examples, and real-world implementation strategies for enterprise deployment.* diff --git a/docs/best-practices/r/r-big-data-processing.md b/docs/best-practices/r/r-big-data-processing.md index 6fd2242..6a3ca76 100644 --- a/docs/best-practices/r/r-big-data-processing.md +++ b/docs/best-practices/r/r-big-data-processing.md @@ -1,6 +1,5 @@ # R Big Data Processing Best Practices -**Objective**: Master senior-level R big data processing patterns for enterprise systems. When you need to process massive datasets efficiently, when you want to leverage distributed computing, when you need enterprise-grade big data workflows—these best practices become your weapon of choice. ## Core Principles @@ -904,4 +903,3 @@ create_big_data_processing_pipeline <- function(pipeline_config) { --- -*This guide provides the complete machinery for implementing big data processing for R applications. Each pattern includes implementation examples, optimization strategies, and real-world usage patterns for enterprise big data systems.* diff --git a/docs/best-practices/r/r-cicd-pipelines.md b/docs/best-practices/r/r-cicd-pipelines.md index 41cfb42..48a3298 100644 --- a/docs/best-practices/r/r-cicd-pipelines.md +++ b/docs/best-practices/r/r-cicd-pipelines.md @@ -1,6 +1,5 @@ # R CI/CD Pipelines Best Practices -**Objective**: Master senior-level R CI/CD pipeline patterns for production systems. When you need to automate testing, building, and deployment, when you want to ensure code quality and reliability, when you need enterprise-grade CI/CD patterns—these best practices become your weapon of choice. ## Core Principles @@ -912,4 +911,3 @@ create_cicd_pipeline <- function(pipeline_config) { --- -*This guide provides the complete machinery for implementing CI/CD pipelines for R applications. Each pattern includes implementation examples, quality control strategies, and real-world usage patterns for enterprise deployment.* diff --git a/docs/best-practices/r/r-containerization.md b/docs/best-practices/r/r-containerization.md index 890da86..cab58c2 100644 --- a/docs/best-practices/r/r-containerization.md +++ b/docs/best-practices/r/r-containerization.md @@ -1,6 +1,5 @@ # R Containerization Best Practices -**Objective**: Master senior-level R containerization patterns for production systems. When you need to package R applications for deployment, when you want to ensure reproducibility across environments, when you need enterprise-grade containerization patterns—these best practices become your weapon of choice. ## Core Principles @@ -1015,4 +1014,3 @@ create_containerization_pipeline <- function(app_config) { --- -*This guide provides the complete machinery for containerizing R applications. Each pattern includes implementation examples, security strategies, and real-world usage patterns for enterprise deployment.* diff --git a/docs/best-practices/r/r-data-analysis-workflows.md b/docs/best-practices/r/r-data-analysis-workflows.md index e3549c5..4a7a5d1 100644 --- a/docs/best-practices/r/r-data-analysis-workflows.md +++ b/docs/best-practices/r/r-data-analysis-workflows.md @@ -1,6 +1,5 @@ # R Data Analysis Workflows Best Practices -**Objective**: Master senior-level R data analysis workflow patterns for production systems. When you need to build reproducible, scalable data analysis pipelines, when you want to follow proven methodologies, when you need enterprise-grade analysis patterns—these best practices become your weapon of choice. ## Core Principles @@ -877,4 +876,3 @@ report_path <- generate_report(results, report_config) --- -*This guide provides the complete machinery for building reproducible data analysis workflows in R. Each pattern includes implementation examples, testing strategies, and real-world usage patterns for enterprise deployment.* diff --git a/docs/best-practices/r/r-data-exploration.md b/docs/best-practices/r/r-data-exploration.md index 7b218da..d3c7799 100644 --- a/docs/best-practices/r/r-data-exploration.md +++ b/docs/best-practices/r/r-data-exploration.md @@ -1,6 +1,5 @@ # R Data Exploration Best Practices -**Objective**: Master senior-level R data exploration patterns for production systems. When you need to understand complex datasets, when you want to discover patterns and insights, when you need enterprise-grade exploration patterns—these best practices become your weapon of choice. ## Core Principles @@ -840,4 +839,3 @@ explore_data <- function(data) { --- -*This guide provides the complete machinery for comprehensive data exploration in R. Each pattern includes implementation examples, visualization strategies, and real-world usage patterns for enterprise deployment.* diff --git a/docs/best-practices/r/r-data-processing.md b/docs/best-practices/r/r-data-processing.md index d178ff9..e91c8f1 100644 --- a/docs/best-practices/r/r-data-processing.md +++ b/docs/best-practices/r/r-data-processing.md @@ -1,6 +1,5 @@ # R Data Processing Best Practices -**Objective**: Master senior-level R data processing patterns for production systems. When you need to process large datasets efficiently, when you want to ensure data quality and reliability, when you need enterprise-grade data processing workflows—these best practices become your weapon of choice. ## Core Principles @@ -1061,4 +1060,3 @@ create_data_processing_pipeline <- function(pipeline_config) { --- -*This guide provides the complete machinery for implementing data processing for R applications. Each pattern includes implementation examples, optimization strategies, and real-world usage patterns for enterprise data processing systems.* diff --git a/docs/best-practices/r/r-database-integration.md b/docs/best-practices/r/r-database-integration.md index fd5a65b..f21dd61 100644 --- a/docs/best-practices/r/r-database-integration.md +++ b/docs/best-practices/r/r-database-integration.md @@ -1,6 +1,5 @@ # R Database Integration Best Practices -**Objective**: Master senior-level R database integration patterns for production systems. When you need to connect R to databases, when you want to ensure data integrity and performance, when you need enterprise-grade database patterns—these best practices become your weapon of choice. ## Core Principles @@ -1080,4 +1079,3 @@ create_database_integration <- function(integration_config) { --- -*This guide provides the complete machinery for implementing database integration for R applications. Each pattern includes implementation examples, optimization strategies, and real-world usage patterns for enterprise database systems.* diff --git a/docs/best-practices/r/r-dev-environment.md b/docs/best-practices/r/r-dev-environment.md index 5a8f3cc..604786e 100644 --- a/docs/best-practices/r/r-dev-environment.md +++ b/docs/best-practices/r/r-dev-environment.md @@ -1,6 +1,5 @@ # R Development Environment Best Practices -**Objective**: Master senior-level R development environment setup and operation across macOS, Linux, and Windows. Copy-paste runnable, auditable, and production-ready. ## Core Principles @@ -555,4 +554,3 @@ goodpractice::gp() # Check quality --- -*This guide provides the complete machinery for setting up a production-ready R development environment. Each pattern includes configuration examples, tooling setup, and real-world implementation strategies for enterprise deployment.* diff --git a/docs/best-practices/r/r-geospatial-analysis.md b/docs/best-practices/r/r-geospatial-analysis.md index 9d2afc8..ff2573b 100644 --- a/docs/best-practices/r/r-geospatial-analysis.md +++ b/docs/best-practices/r/r-geospatial-analysis.md @@ -1,6 +1,5 @@ # R Geospatial Analysis Best Practices -**Objective**: Master senior-level R geospatial analysis patterns for production systems. When you need to analyze spatial data, when you want to build location-aware applications, when you need enterprise-grade geospatial patterns—these best practices become your weapon of choice. ## Core Principles @@ -977,4 +976,3 @@ spatial_analysis_pipeline <- function(data_path, analysis_config) { --- -*This guide provides the complete machinery for comprehensive geospatial analysis in R. Each pattern includes implementation examples, visualization strategies, and real-world usage patterns for enterprise deployment.* diff --git a/docs/best-practices/r/r-interactive-applications.md b/docs/best-practices/r/r-interactive-applications.md index c1edf21..9f7a346 100644 --- a/docs/best-practices/r/r-interactive-applications.md +++ b/docs/best-practices/r/r-interactive-applications.md @@ -1,6 +1,5 @@ # R Interactive Applications Best Practices -**Objective**: Master senior-level R interactive application patterns for production systems. When you need to build dynamic, user-friendly applications, when you want to create engaging data experiences, when you need enterprise-grade interactive patterns—these best practices become your weapon of choice. ## Core Principles @@ -1083,4 +1082,3 @@ create_interactive_application <- function(data, app_config) { --- -*This guide provides the complete machinery for building interactive applications in R. Each pattern includes implementation examples, optimization strategies, and real-world usage patterns for enterprise deployment.* diff --git a/docs/best-practices/r/r-machine-learning.md b/docs/best-practices/r/r-machine-learning.md index b186bb9..c39b0a6 100644 --- a/docs/best-practices/r/r-machine-learning.md +++ b/docs/best-practices/r/r-machine-learning.md @@ -1,6 +1,5 @@ # R Machine Learning Best Practices -**Objective**: Master senior-level R machine learning patterns for production systems. When you need to build robust, scalable ML models, when you want to follow best practices for model development and deployment, when you need enterprise-grade ML patterns—these best practices become your weapon of choice. ## Core Principles @@ -996,4 +995,3 @@ ml_pipeline <- function(data, target, model_config) { --- -*This guide provides the complete machinery for building production-ready machine learning systems in R. Each pattern includes implementation examples, validation strategies, and real-world usage patterns for enterprise deployment.* diff --git a/docs/best-practices/r/r-memory-management.md b/docs/best-practices/r/r-memory-management.md index 983f4f8..b945f9c 100644 --- a/docs/best-practices/r/r-memory-management.md +++ b/docs/best-practices/r/r-memory-management.md @@ -1,6 +1,5 @@ # R Memory Management Best Practices -**Objective**: Master senior-level R memory management patterns for production systems. When you need to handle large datasets efficiently, when you want to optimize memory usage, when you need enterprise-grade memory patterns—these best practices become your weapon of choice. ## Core Principles @@ -930,4 +929,3 @@ manage_memory <- function(data, memory_config) { --- -*This guide provides the complete machinery for managing memory efficiently in R. Each pattern includes implementation examples, monitoring strategies, and real-world usage patterns for enterprise deployment.* diff --git a/docs/best-practices/r/r-package-development.md b/docs/best-practices/r/r-package-development.md index dd7c26f..0f91ed1 100644 --- a/docs/best-practices/r/r-package-development.md +++ b/docs/best-practices/r/r-package-development.md @@ -1,6 +1,5 @@ # R Package Development Best Practices -**Objective**: Master senior-level R package development patterns for production systems. When you need to build robust, maintainable R packages, when you want to follow modern development practices, when you need enterprise-grade package patterns—these best practices become your weapon of choice. ## Core Principles @@ -644,4 +643,3 @@ print.myclass <- function(x, ...) { --- -*This guide provides the complete machinery for building production-ready R packages. Each pattern includes implementation examples, testing strategies, and real-world usage patterns for enterprise deployment.* diff --git a/docs/best-practices/r/r-parallel-computing.md b/docs/best-practices/r/r-parallel-computing.md index 134c672..e095605 100644 --- a/docs/best-practices/r/r-parallel-computing.md +++ b/docs/best-practices/r/r-parallel-computing.md @@ -1,6 +1,5 @@ # R Parallel Computing Best Practices -**Objective**: Master senior-level R parallel computing patterns for production systems. When you need to leverage multiple cores for computation, when you want to scale R applications, when you need enterprise-grade parallel patterns—these best practices become your weapon of choice. ## Core Principles @@ -1076,4 +1075,3 @@ parallel_computing_pipeline <- function(data, computing_config) { --- -*This guide provides the complete machinery for implementing parallel computing in R. Each pattern includes implementation examples, optimization strategies, and real-world usage patterns for enterprise deployment.* diff --git a/docs/best-practices/r/r-performance-tuning.md b/docs/best-practices/r/r-performance-tuning.md index de2c474..df56754 100644 --- a/docs/best-practices/r/r-performance-tuning.md +++ b/docs/best-practices/r/r-performance-tuning.md @@ -1,6 +1,5 @@ # R Performance Tuning Best Practices -**Objective**: Master senior-level R performance tuning patterns for production systems. When you need to optimize R code for speed and efficiency, when you want to handle large datasets effectively, when you need enterprise-grade performance patterns—these best practices become your weapon of choice. ## Core Principles @@ -1033,4 +1032,3 @@ optimize_performance <- function(code, data, optimization_config) { --- -*This guide provides the complete machinery for optimizing R code performance. Each pattern includes implementation examples, profiling strategies, and real-world usage patterns for enterprise deployment.* diff --git a/docs/best-practices/r/r-production-deployment.md b/docs/best-practices/r/r-production-deployment.md index d337b13..ad9264e 100644 --- a/docs/best-practices/r/r-production-deployment.md +++ b/docs/best-practices/r/r-production-deployment.md @@ -1,6 +1,5 @@ # R Production Deployment Best Practices -**Objective**: Master senior-level R production deployment patterns for enterprise systems. When you need to deploy R applications to production, when you want to ensure reliability and scalability, when you need enterprise-grade deployment strategies—these best practices become your weapon of choice. ## Core Principles @@ -1238,4 +1237,3 @@ create_production_deployment <- function(deployment_config) { --- -*This guide provides the complete machinery for implementing production deployment for R applications. Each pattern includes implementation examples, deployment strategies, and real-world usage patterns for enterprise deployment.* diff --git a/docs/best-practices/r/r-reporting-workflows.md b/docs/best-practices/r/r-reporting-workflows.md index 7df3265..27bbd69 100644 --- a/docs/best-practices/r/r-reporting-workflows.md +++ b/docs/best-practices/r/r-reporting-workflows.md @@ -1,6 +1,5 @@ # R Reporting Workflows Best Practices -**Objective**: Master senior-level R reporting workflow patterns for production systems. When you need to create automated, reproducible reports, when you want to follow best practices for document generation, when you need enterprise-grade reporting patterns—these best practices become your weapon of choice. ## Core Principles @@ -1009,4 +1008,3 @@ create_reporting_workflow <- function(data, workflow_config) { --- -*This guide provides the complete machinery for creating automated, reproducible reports in R. Each pattern includes implementation examples, quality control strategies, and real-world usage patterns for enterprise deployment.* diff --git a/docs/best-practices/r/r-statistical-modeling.md b/docs/best-practices/r/r-statistical-modeling.md index 721f828..0a58964 100644 --- a/docs/best-practices/r/r-statistical-modeling.md +++ b/docs/best-practices/r/r-statistical-modeling.md @@ -1,6 +1,5 @@ # R Statistical Modeling Best Practices -**Objective**: Master senior-level R statistical modeling patterns for production systems. When you need to build robust, interpretable statistical models, when you want to follow best practices for model selection and validation, when you need enterprise-grade modeling patterns—these best practices become your weapon of choice. ## Core Principles @@ -945,4 +944,3 @@ fit_and_validate <- function(data, formula, model_type = "linear") { --- -*This guide provides the complete machinery for building robust statistical models in R. Each pattern includes implementation examples, validation strategies, and real-world usage patterns for enterprise deployment.* diff --git a/docs/best-practices/r/r-testing-best-practices.md b/docs/best-practices/r/r-testing-best-practices.md index eda4a02..4235449 100644 --- a/docs/best-practices/r/r-testing-best-practices.md +++ b/docs/best-practices/r/r-testing-best-practices.md @@ -1,6 +1,5 @@ # R Testing Best Practices -**Objective**: Master senior-level R testing patterns for production systems. When you need to build robust, reliable R applications, when you want to ensure code quality, when you need enterprise-grade testing patterns—these best practices become your weapon of choice. ## Core Principles @@ -672,4 +671,3 @@ test_that("function handles different inputs", { --- -*This guide provides the complete machinery for building comprehensive test suites in R. Each pattern includes implementation examples, testing strategies, and real-world usage patterns for enterprise deployment.* diff --git a/docs/best-practices/r/r-visualization-best-practices.md b/docs/best-practices/r/r-visualization-best-practices.md index 85d8e62..193a9b2 100644 --- a/docs/best-practices/r/r-visualization-best-practices.md +++ b/docs/best-practices/r/r-visualization-best-practices.md @@ -1,6 +1,5 @@ # R Visualization Best Practices -**Objective**: Master senior-level R visualization patterns for production systems. When you need to create compelling, informative visualizations, when you want to follow design best practices, when you need enterprise-grade visualization patterns—these best practices become your weapon of choice. ## Core Principles @@ -1050,4 +1049,3 @@ create_visualization_pipeline <- function(data, viz_config) { --- -*This guide provides the complete machinery for creating compelling visualizations in R. Each pattern includes implementation examples, design strategies, and real-world usage patterns for enterprise deployment.* diff --git a/docs/best-practices/rust/index.md b/docs/best-practices/rust/index.md index d2fa274..e512dea 100644 --- a/docs/best-practices/rust/index.md +++ b/docs/best-practices/rust/index.md @@ -1,6 +1,5 @@ # Rust Development Best Practices -**Objective**: Master senior-level Rust development patterns for production systems. When you need to build robust, scalable Rust applications, when you want to follow proven methodologies, when you need enterprise-grade patterns—these best practices become your weapon of choice. ## Core Development @@ -55,4 +54,3 @@ --- -*These best practices provide the complete machinery for building production-ready Rust systems. Each guide includes architectural patterns, configuration examples, and real-world implementation strategies for enterprise deployment.* diff --git a/docs/best-practices/rust/rust-api-design.md b/docs/best-practices/rust/rust-api-design.md index 07ab484..fef42e3 100644 --- a/docs/best-practices/rust/rust-api-design.md +++ b/docs/best-practices/rust/rust-api-design.md @@ -1,6 +1,5 @@ # Rust API Design Best Practices -**Objective**: Master senior-level Rust API design patterns for production systems. When you need to build robust, scalable APIs, when you want to create maintainable interfaces, when you need enterprise-grade API design—these best practices become your weapon of choice. ## Core Principles @@ -809,4 +808,3 @@ pub fn setup_rust_api_design() { --- -*This guide provides the complete machinery for Rust API design. Each pattern includes implementation examples, API strategies, and real-world usage patterns for enterprise API development.* diff --git a/docs/best-practices/rust/rust-caching-strategies.md b/docs/best-practices/rust/rust-caching-strategies.md index 99209fa..6429c09 100644 --- a/docs/best-practices/rust/rust-caching-strategies.md +++ b/docs/best-practices/rust/rust-caching-strategies.md @@ -1,6 +1,5 @@ # Rust Caching Strategies Best Practices -**Objective**: Master senior-level Rust caching patterns for production systems. When you need to implement high-performance caching, when you want to optimize data access patterns, when you need enterprise-grade caching strategies—these best practices become your weapon of choice. ## Core Principles @@ -982,4 +981,3 @@ pub fn setup_rust_caching_strategies() { --- -*This guide provides the complete machinery for Rust caching strategies. Each pattern includes implementation examples, caching strategies, and real-world usage patterns for enterprise caching systems.* diff --git a/docs/best-practices/rust/rust-cicd-pipelines.md b/docs/best-practices/rust/rust-cicd-pipelines.md index 665ebca..ac983de 100644 --- a/docs/best-practices/rust/rust-cicd-pipelines.md +++ b/docs/best-practices/rust/rust-cicd-pipelines.md @@ -1,6 +1,5 @@ # Rust CI/CD Pipelines Best Practices -**Objective**: Master senior-level Rust CI/CD pipeline patterns for production systems. When you need to build automated testing and deployment pipelines, when you want to ensure code quality and reliability, when you need enterprise-grade CI/CD—these best practices become your weapon of choice. ## Core Principles @@ -1041,4 +1040,3 @@ pub fn setup_rust_cicd() { --- -*This guide provides the complete machinery for Rust CI/CD pipelines. Each pattern includes implementation examples, pipeline strategies, and real-world usage patterns for enterprise CI/CD systems.* diff --git a/docs/best-practices/rust/rust-code-quality.md b/docs/best-practices/rust/rust-code-quality.md index fc6f461..9b293d6 100644 --- a/docs/best-practices/rust/rust-code-quality.md +++ b/docs/best-practices/rust/rust-code-quality.md @@ -1,6 +1,5 @@ # Rust Code Quality Best Practices -**Objective**: Master senior-level Rust code quality patterns for production systems. When you need to maintain high code quality, when you want to enforce coding standards, when you need enterprise-grade quality assurance—these best practices become your weapon of choice. ## Core Principles @@ -950,4 +949,3 @@ pub fn setup_rust_code_quality() { --- -*This guide provides the complete machinery for Rust code quality best practices. Each pattern includes implementation examples, quality standards, and real-world usage patterns for enterprise code development.* diff --git a/docs/best-practices/rust/rust-compliance.md b/docs/best-practices/rust/rust-compliance.md index 00052a1..efa631d 100644 --- a/docs/best-practices/rust/rust-compliance.md +++ b/docs/best-practices/rust/rust-compliance.md @@ -1,6 +1,5 @@ # Rust Compliance Best Practices -**Objective**: Master senior-level Rust compliance patterns for production systems. When you need to implement regulatory compliance, when you want to ensure data protection, when you need enterprise-grade compliance strategies—these best practices become your weapon of choice. ## Core Principles @@ -531,4 +530,3 @@ pub fn setup_rust_compliance() { --- -*This guide provides the complete machinery for Rust compliance. Each pattern includes implementation examples, compliance strategies, and real-world usage patterns for enterprise compliance systems.* diff --git a/docs/best-practices/rust/rust-concurrency-patterns.md b/docs/best-practices/rust/rust-concurrency-patterns.md index 0c65e7f..4de6403 100644 --- a/docs/best-practices/rust/rust-concurrency-patterns.md +++ b/docs/best-practices/rust/rust-concurrency-patterns.md @@ -1,6 +1,5 @@ # Rust Concurrency Patterns Best Practices -**Objective**: Master senior-level Rust concurrency patterns for production systems. When you need to build concurrent and parallel applications, when you want to leverage multiple cores effectively, when you need enterprise-grade concurrency strategies—these best practices become your weapon of choice. ## Core Principles @@ -854,4 +853,3 @@ pub fn setup_rust_concurrency() { --- -*This guide provides the complete machinery for Rust concurrency patterns. Each pattern includes implementation examples, concurrency strategies, and real-world usage patterns for enterprise parallel processing.* diff --git a/docs/best-practices/rust/rust-containerization.md b/docs/best-practices/rust/rust-containerization.md index 45f054b..7d453ab 100644 --- a/docs/best-practices/rust/rust-containerization.md +++ b/docs/best-practices/rust/rust-containerization.md @@ -1,6 +1,5 @@ # Rust Containerization Best Practices -**Objective**: Master senior-level Rust containerization patterns for production systems. When you need to build efficient Docker containers, when you want to optimize Rust applications for containers, when you need enterprise-grade containerization—these best practices become your weapon of choice. ## Core Principles @@ -1019,4 +1018,3 @@ pub fn setup_rust_containerization() { --- -*This guide provides the complete machinery for Rust containerization. Each pattern includes implementation examples, container strategies, and real-world usage patterns for enterprise containerized systems.* diff --git a/docs/best-practices/rust/rust-data-analysis.md b/docs/best-practices/rust/rust-data-analysis.md index 9151a1f..b6494db 100644 --- a/docs/best-practices/rust/rust-data-analysis.md +++ b/docs/best-practices/rust/rust-data-analysis.md @@ -1,6 +1,5 @@ # Rust Data Analysis Best Practices -**Objective**: Master senior-level Rust data analysis patterns for production systems. When you need to analyze large datasets efficiently, when you want to leverage Rust's performance for data analysis, when you need enterprise-grade data analysis—these best practices become your weapon of choice. ## Core Principles @@ -908,4 +907,3 @@ pub fn setup_rust_data_analysis() { --- -*This guide provides the complete machinery for Rust data analysis. Each pattern includes implementation examples, analysis strategies, and real-world usage patterns for enterprise data analysis.* diff --git a/docs/best-practices/rust/rust-data-processing.md b/docs/best-practices/rust/rust-data-processing.md index 9964dbb..1a12f78 100644 --- a/docs/best-practices/rust/rust-data-processing.md +++ b/docs/best-practices/rust/rust-data-processing.md @@ -1,6 +1,5 @@ # Rust Data Processing Best Practices -**Objective**: Master senior-level Rust data processing patterns for production systems. When you need to build high-performance data pipelines, when you want to process large datasets efficiently, when you need enterprise-grade data processing—these best practices become your weapon of choice. ## Core Principles @@ -836,4 +835,3 @@ pub fn setup_rust_data_processing() { --- -*This guide provides the complete machinery for Rust data processing. Each pattern includes implementation examples, data processing strategies, and real-world usage patterns for enterprise data pipelines.* diff --git a/docs/best-practices/rust/rust-data-storage.md b/docs/best-practices/rust/rust-data-storage.md index ba2e0bb..863080d 100644 --- a/docs/best-practices/rust/rust-data-storage.md +++ b/docs/best-practices/rust/rust-data-storage.md @@ -1,6 +1,5 @@ # Rust Data Storage Best Practices -**Objective**: Master senior-level Rust data storage patterns for production systems. When you need to implement robust data persistence, when you want to optimize storage performance, when you need enterprise-grade data storage strategies—these best practices become your weapon of choice. ## Core Principles @@ -1106,4 +1105,3 @@ pub fn setup_rust_data_storage() { --- -*This guide provides the complete machinery for Rust data storage. Each pattern includes implementation examples, storage strategies, and real-world usage patterns for enterprise data storage systems.* diff --git a/docs/best-practices/rust/rust-database-patterns.md b/docs/best-practices/rust/rust-database-patterns.md index 30aa7f7..3c602d1 100644 --- a/docs/best-practices/rust/rust-database-patterns.md +++ b/docs/best-practices/rust/rust-database-patterns.md @@ -1,6 +1,5 @@ # Rust Database Patterns Best Practices -**Objective**: Master senior-level Rust database patterns for production systems. When you need to build robust database applications, when you want to implement efficient data access patterns, when you need enterprise-grade database strategies—these best practices become your weapon of choice. ## Core Principles @@ -1003,4 +1002,3 @@ pub fn setup_rust_database() { --- -*This guide provides the complete machinery for Rust database patterns. Each pattern includes implementation examples, database strategies, and real-world usage patterns for enterprise database systems.* diff --git a/docs/best-practices/rust/rust-error-handling.md b/docs/best-practices/rust/rust-error-handling.md index 68b6c5d..b2bc1d1 100644 --- a/docs/best-practices/rust/rust-error-handling.md +++ b/docs/best-practices/rust/rust-error-handling.md @@ -1,6 +1,5 @@ # Rust Error Handling Best Practices -**Objective**: Master senior-level Rust error handling patterns for production systems. When you need to build robust error handling, when you want to create maintainable error propagation, when you need enterprise-grade error management—these best practices become your weapon of choice. ## Core Principles @@ -857,4 +856,3 @@ pub fn setup_rust_error_handling() { --- -*This guide provides the complete machinery for Rust error handling. Each pattern includes implementation examples, error strategies, and real-world usage patterns for enterprise error management.* diff --git a/docs/best-practices/rust/rust-generics-traits.md b/docs/best-practices/rust/rust-generics-traits.md index a71b700..fa44c2a 100644 --- a/docs/best-practices/rust/rust-generics-traits.md +++ b/docs/best-practices/rust/rust-generics-traits.md @@ -1,6 +1,5 @@ # Rust Generics & Traits Best Practices -**Objective**: Master senior-level Rust generics and traits patterns for production systems. When you need to build reusable, type-safe code, when you want to leverage Rust's type system, when you need enterprise-grade generic programming—these best practices become your weapon of choice. ## Core Principles @@ -460,4 +459,3 @@ pub fn setup_rust_generics_traits() { --- -*This guide provides the complete machinery for Rust generics and traits. Each pattern includes implementation examples, type system strategies, and real-world usage patterns for enterprise generic programming.* diff --git a/docs/best-practices/rust/rust-geospatial-development.md b/docs/best-practices/rust/rust-geospatial-development.md index d796a88..e08b04c 100644 --- a/docs/best-practices/rust/rust-geospatial-development.md +++ b/docs/best-practices/rust/rust-geospatial-development.md @@ -1,6 +1,5 @@ # Rust Geospatial Development Best Practices -**Objective**: Master senior-level Rust geospatial development patterns for production systems. When you need to build high-performance geospatial applications, when you want to leverage Rust's speed for spatial data processing, when you need enterprise-grade geospatial patterns—these best practices become your weapon of choice. ## Core Principles @@ -953,4 +952,3 @@ pub fn setup_rust_geospatial() { --- -*This guide provides the complete machinery for Rust geospatial development. Each pattern includes implementation examples, spatial strategies, and real-world usage patterns for enterprise geospatial systems.* diff --git a/docs/best-practices/rust/rust-machine-learning.md b/docs/best-practices/rust/rust-machine-learning.md index 7e02168..ce9e1d3 100644 --- a/docs/best-practices/rust/rust-machine-learning.md +++ b/docs/best-practices/rust/rust-machine-learning.md @@ -1,6 +1,5 @@ # Rust Machine Learning Best Practices -**Objective**: Master senior-level Rust machine learning patterns for production systems. When you need to build high-performance ML models, when you want to leverage Rust's speed for ML workloads, when you need enterprise-grade ML patterns—these best practices become your weapon of choice. ## Core Principles @@ -990,4 +989,3 @@ pub fn setup_rust_machine_learning() { --- -*This guide provides the complete machinery for Rust machine learning. Each pattern includes implementation examples, ML strategies, and real-world usage patterns for enterprise ML systems.* diff --git a/docs/best-practices/rust/rust-memory-management.md b/docs/best-practices/rust/rust-memory-management.md index 0688c35..9f1a6ff 100644 --- a/docs/best-practices/rust/rust-memory-management.md +++ b/docs/best-practices/rust/rust-memory-management.md @@ -1,6 +1,5 @@ # Rust Memory Management Best Practices -**Objective**: Master senior-level Rust memory management patterns for production systems. When you need to optimize memory usage, when you want to understand Rust's memory model, when you need enterprise-grade memory strategies—these best practices become your weapon of choice. ## Core Principles @@ -703,4 +702,3 @@ pub fn setup_rust_memory_management() { --- -*This guide provides the complete machinery for Rust memory management. Each pattern includes implementation examples, memory strategies, and real-world usage patterns for enterprise memory optimization.* diff --git a/docs/best-practices/rust/rust-microservices.md b/docs/best-practices/rust/rust-microservices.md index 11c36cb..c6f175f 100644 --- a/docs/best-practices/rust/rust-microservices.md +++ b/docs/best-practices/rust/rust-microservices.md @@ -1,6 +1,5 @@ # Rust Microservices Best Practices -**Objective**: Master senior-level Rust microservices patterns for production systems. When you need to build scalable microservices, when you want to leverage Rust's performance for distributed systems, when you need enterprise-grade microservice patterns—these best practices become your weapon of choice. ## Core Principles @@ -697,4 +696,3 @@ pub fn setup_rust_microservices() { --- -*This guide provides the complete machinery for Rust microservices. Each pattern includes implementation examples, microservice strategies, and real-world usage patterns for enterprise distributed systems.* diff --git a/docs/best-practices/rust/rust-monitoring-observability.md b/docs/best-practices/rust/rust-monitoring-observability.md index 3222db0..f732197 100644 --- a/docs/best-practices/rust/rust-monitoring-observability.md +++ b/docs/best-practices/rust/rust-monitoring-observability.md @@ -1,6 +1,5 @@ # Rust Monitoring & Observability Best Practices -**Objective**: Master senior-level Rust monitoring and observability patterns for production systems. When you need to build comprehensive monitoring, when you want to implement distributed tracing, when you need enterprise-grade observability—these best practices become your weapon of choice. ## Core Principles @@ -1159,4 +1158,3 @@ pub fn setup_rust_monitoring() { --- -*This guide provides the complete machinery for Rust monitoring and observability. Each pattern includes implementation examples, monitoring strategies, and real-world usage patterns for enterprise observability systems.* diff --git a/docs/best-practices/rust/rust-ownership-borrowing.md b/docs/best-practices/rust/rust-ownership-borrowing.md index a7ebc48..f5a0b02 100644 --- a/docs/best-practices/rust/rust-ownership-borrowing.md +++ b/docs/best-practices/rust/rust-ownership-borrowing.md @@ -1,6 +1,5 @@ # Rust Ownership & Borrowing Best Practices -**Objective**: Master senior-level Rust ownership and borrowing patterns for production systems. When you need to understand Rust's memory safety guarantees, when you want to write efficient and safe code, when you need enterprise-grade ownership patterns—these best practices become your weapon of choice. ## Core Principles @@ -797,4 +796,3 @@ pub fn setup_rust_ownership_borrowing() { --- -*This guide provides the complete machinery for Rust ownership and borrowing. Each pattern includes implementation examples, ownership strategies, and real-world usage patterns for enterprise memory safety.* diff --git a/docs/best-practices/rust/rust-package-development.md b/docs/best-practices/rust/rust-package-development.md index cea2b53..9e1fc65 100644 --- a/docs/best-practices/rust/rust-package-development.md +++ b/docs/best-practices/rust/rust-package-development.md @@ -1,6 +1,5 @@ # Rust Package Development Best Practices -**Objective**: Master senior-level Rust package development patterns for production systems. When you need to build and ship Rust crates, when you want to follow proven methodologies, when you need enterprise-grade packaging strategies—these best practices become your weapon of choice. ## Core Principles @@ -866,4 +865,3 @@ pub fn setup_rust_package() { --- -*This guide provides the complete machinery for Rust package development. Each pattern includes implementation examples, testing strategies, and real-world usage patterns for enterprise crate development.* diff --git a/docs/best-practices/rust/rust-performance-tuning.md b/docs/best-practices/rust/rust-performance-tuning.md index 5dc6c66..6da22e9 100644 --- a/docs/best-practices/rust/rust-performance-tuning.md +++ b/docs/best-practices/rust/rust-performance-tuning.md @@ -1,6 +1,5 @@ # Rust Performance Tuning Best Practices -**Objective**: Master senior-level Rust performance optimization patterns for production systems. When you need to optimize for speed and memory efficiency, when you want to build high-performance applications, when you need enterprise-grade performance strategies—these best practices become your weapon of choice. ## Core Principles @@ -890,4 +889,3 @@ pub fn setup_rust_performance() { --- -*This guide provides the complete machinery for Rust performance tuning. Each pattern includes implementation examples, optimization strategies, and real-world usage patterns for enterprise performance optimization.* diff --git a/docs/best-practices/rust/rust-production-deployment.md b/docs/best-practices/rust/rust-production-deployment.md index 3018708..0d0a6ca 100644 --- a/docs/best-practices/rust/rust-production-deployment.md +++ b/docs/best-practices/rust/rust-production-deployment.md @@ -1,6 +1,5 @@ # Rust Production Deployment Best Practices -**Objective**: Master senior-level Rust production deployment patterns for enterprise systems. When you need to deploy Rust applications to production, when you want to ensure high availability and reliability, when you need enterprise-grade deployment strategies—these best practices become your weapon of choice. ## Core Principles @@ -1021,4 +1020,3 @@ pub fn setup_rust_production_deployment() { --- -*This guide provides the complete machinery for Rust production deployment. Each pattern includes implementation examples, deployment strategies, and real-world usage patterns for enterprise production systems.* diff --git a/docs/best-practices/rust/rust-secrets-management.md b/docs/best-practices/rust/rust-secrets-management.md index 1199e9c..ffac0d3 100644 --- a/docs/best-practices/rust/rust-secrets-management.md +++ b/docs/best-practices/rust/rust-secrets-management.md @@ -1,6 +1,5 @@ # Rust Secrets Management Best Practices -**Objective**: Master senior-level Rust secrets management patterns for production systems. When you need to handle sensitive data securely, when you want to implement robust secret rotation, when you need enterprise-grade secrets management strategies—these best practices become your weapon of choice. ## Core Principles @@ -997,4 +996,3 @@ pub fn setup_rust_secrets_management() { --- -*This guide provides the complete machinery for Rust secrets management. Each pattern includes implementation examples, security strategies, and real-world usage patterns for enterprise secrets management systems.* diff --git a/docs/best-practices/rust/rust-security-best-practices.md b/docs/best-practices/rust/rust-security-best-practices.md index 045e6e5..74c035f 100644 --- a/docs/best-practices/rust/rust-security-best-practices.md +++ b/docs/best-practices/rust/rust-security-best-practices.md @@ -1,6 +1,5 @@ # Rust Security Best Practices -**Objective**: Master senior-level Rust security patterns for production systems. When you need to implement robust security measures, when you want to protect against common vulnerabilities, when you need enterprise-grade security strategies—these best practices become your weapon of choice. ## Core Principles @@ -1026,4 +1025,3 @@ pub fn setup_rust_security() { --- -*This guide provides the complete machinery for Rust security. Each pattern includes implementation examples, security strategies, and real-world usage patterns for enterprise security systems.* diff --git a/docs/best-practices/rust/rust-testing-best-practices.md b/docs/best-practices/rust/rust-testing-best-practices.md index 7e9be15..ba4872e 100644 --- a/docs/best-practices/rust/rust-testing-best-practices.md +++ b/docs/best-practices/rust/rust-testing-best-practices.md @@ -1,6 +1,5 @@ # Rust Testing Best Practices -**Objective**: Master senior-level Rust testing patterns for production systems. When you need to build comprehensive test suites, when you want to ensure code reliability, when you need enterprise-grade testing strategies—these best practices become your weapon of choice. ## Core Principles @@ -888,4 +887,3 @@ pub fn setup_rust_testing() { --- -*This guide provides the complete machinery for Rust testing best practices. Each pattern includes implementation examples, testing strategies, and real-world usage patterns for enterprise test development.* diff --git a/docs/best-practices/rust/rust-unsafe-programming.md b/docs/best-practices/rust/rust-unsafe-programming.md index c78f883..f0f4f76 100644 --- a/docs/best-practices/rust/rust-unsafe-programming.md +++ b/docs/best-practices/rust/rust-unsafe-programming.md @@ -1,6 +1,5 @@ # Rust Unsafe Programming Best Practices -**Objective**: Master senior-level Rust unsafe programming patterns for production systems. When you need to interface with C code, when you want to optimize performance, when you need enterprise-grade unsafe patterns—these best practices become your weapon of choice. ## Core Principles @@ -618,4 +617,3 @@ pub fn setup_rust_unsafe() { --- -*This guide provides the complete machinery for Rust unsafe programming. Each pattern includes implementation examples, safety strategies, and real-world usage patterns for enterprise unsafe code.* diff --git a/docs/best-practices/rust/rust-web-performance.md b/docs/best-practices/rust/rust-web-performance.md index ebc2c00..eae18bd 100644 --- a/docs/best-practices/rust/rust-web-performance.md +++ b/docs/best-practices/rust/rust-web-performance.md @@ -1,6 +1,5 @@ # Rust Web Performance Best Practices -**Objective**: Master senior-level Rust web performance optimization patterns for production systems. When you need to build high-performance web applications, when you want to optimize for speed and scalability, when you need enterprise-grade performance strategies—these best practices become your weapon of choice. ## Core Principles @@ -796,4 +795,3 @@ pub fn setup_rust_web_performance() { --- -*This guide provides the complete machinery for Rust web performance. Each pattern includes implementation examples, performance strategies, and real-world usage patterns for enterprise web optimization.* diff --git a/docs/best-practices/rust/rust-web-services.md b/docs/best-practices/rust/rust-web-services.md index ab26ab0..5aa6348 100644 --- a/docs/best-practices/rust/rust-web-services.md +++ b/docs/best-practices/rust/rust-web-services.md @@ -1,6 +1,5 @@ # Rust Web Services Best Practices -**Objective**: Master senior-level Rust web service patterns for production systems. When you need to build scalable web services, when you want to leverage Rust's performance for web applications, when you need enterprise-grade web service patterns—these best practices become your weapon of choice. ## Core Principles @@ -674,4 +673,3 @@ pub fn setup_rust_web_services() { --- -*This guide provides the complete machinery for Rust web services. Each pattern includes implementation examples, web service strategies, and real-world usage patterns for enterprise web development.* diff --git a/docs/best-practices/security/encryption-lifecycle-and-crypto-rotation.md b/docs/best-practices/security/encryption-lifecycle-and-crypto-rotation.md index 533c699..149348e 100644 --- a/docs/best-practices/security/encryption-lifecycle-and-crypto-rotation.md +++ b/docs/best-practices/security/encryption-lifecycle-and-crypto-rotation.md @@ -1,6 +1,5 @@ # Secret Supply Chains, Encryption Lifecycle Management & Cryptographic Rotation Strategy: Best Practices -**Objective**: Establish comprehensive encryption lifecycle governance that manages key rotation, cryptographic agility, and secret supply chains across data in motion, data at rest, and application-layer encryption. When you need encryption governance, when you want key rotation, when you need crypto agility—this guide provides the complete framework. ## Introduction diff --git a/docs/best-practices/security/iam-rbac-abac-governance.md b/docs/best-practices/security/iam-rbac-abac-governance.md index d149888..da91546 100644 --- a/docs/best-practices/security/iam-rbac-abac-governance.md +++ b/docs/best-practices/security/iam-rbac-abac-governance.md @@ -6,7 +6,6 @@ tags: # Identity & Access Management, RBAC/ABAC, and Least-Privilege Governance for Distributed Systems -**Objective**: Master production-grade identity and access management across heterogeneous distributed systems. When you need to implement least-privilege access, RBAC/ABAC patterns, SSO integration, and comprehensive authorization governance—this guide provides complete patterns and implementations. ## Introduction diff --git a/docs/best-practices/security/identity-federation-authz-authn-architecture.md b/docs/best-practices/security/identity-federation-authz-authn-architecture.md index 408c8f2..b9a312d 100644 --- a/docs/best-practices/security/identity-federation-authz-authn-architecture.md +++ b/docs/best-practices/security/identity-federation-authz-authn-architecture.md @@ -1,6 +1,5 @@ # Cross-Domain Identity Federation, AuthZ/AuthN Architecture & Identity Propagation Models: Best Practices -**Objective**: Establish comprehensive identity federation architecture that unifies authentication and authorization across Rancher, RKE2 clusters, FastAPI/NiceGUI services, Postgres/FDWs, MLflow, object stores, and multi-environment deployments. When you need unified identity, when you want identity propagation, when you need least-privilege governance—this guide provides the complete framework. ## Introduction diff --git a/docs/best-practices/security/index.md b/docs/best-practices/security/index.md index 4d103b5..4aede71 100644 --- a/docs/best-practices/security/index.md +++ b/docs/best-practices/security/index.md @@ -1,6 +1,5 @@ # Security Best Practices -**Objective**: Master production-grade security patterns for infrastructure, applications, and data. When you need to secure credentials, protect data, harden systems, and maintain compliance—these security best practices become your foundation. This collection provides comprehensive guides for secrets management, key rotation, authentication, authorization, and security operations. Each guide includes architectural patterns, configuration examples, and real-world implementation strategies. diff --git a/docs/best-practices/security/secrets-governance.md b/docs/best-practices/security/secrets-governance.md index c5c9b47..840dadc 100644 --- a/docs/best-practices/security/secrets-governance.md +++ b/docs/best-practices/security/secrets-governance.md @@ -1,6 +1,5 @@ # End-to-End Secrets Management & Key Rotation Governance: Best Practices for Infrastructure, Applications, and CI/CD -**Objective**: Master production-grade secrets management across hybrid multi-cloud and air-gapped environments. When you need to secure credentials, rotate keys safely, inject secrets into workloads, and maintain compliance—this guide provides complete patterns and automation. ## Introduction diff --git a/docs/best-practices/security/secure-by-design-polyglot.md b/docs/best-practices/security/secure-by-design-polyglot.md index ec6bd5e..0d399fa 100644 --- a/docs/best-practices/security/secure-by-design-polyglot.md +++ b/docs/best-practices/security/secure-by-design-polyglot.md @@ -1,6 +1,5 @@ # Secure-by-Design Lifecycle Architecture Across Polyglot Systems: Best Practices -**Objective**: Establish security as a lifecycle concern, not a patch, across Python, Go, Rust, and all infrastructure. When you need secure defaults, when you want zero-trust communication, when you need code-to-cloud traceability—this guide provides the complete framework. ## Introduction diff --git a/docs/best-practices/security/secure-sandboxing-and-multi-tenant-isolation.md b/docs/best-practices/security/secure-sandboxing-and-multi-tenant-isolation.md index 03174c2..2ca78ca 100644 --- a/docs/best-practices/security/secure-sandboxing-and-multi-tenant-isolation.md +++ b/docs/best-practices/security/secure-sandboxing-and-multi-tenant-isolation.md @@ -1,6 +1,5 @@ # Secure Computes, Sandboxing, and Multi-Tenant Isolation for Polyglot Systems: Best Practices -**Objective**: Establish comprehensive sandboxing and multi-tenant isolation patterns across Python, Rust, Go microservices, ML inference, embedded engines, and containerized workloads. When you need secure isolation, when you want multi-tenant safety, when you need zero-trust internal boundaries—this guide provides the complete framework. ## Introduction diff --git a/docs/best-practices/testing/end-to-end-testing-strategy.md b/docs/best-practices/testing/end-to-end-testing-strategy.md index a5760f9..1e945ed 100644 --- a/docs/best-practices/testing/end-to-end-testing-strategy.md +++ b/docs/best-practices/testing/end-to-end-testing-strategy.md @@ -1,6 +1,5 @@ # End-to-End Testing, Integration Validation, and QA Strategy for Distributed Systems -**Objective**: Master production-grade testing strategies for distributed, multi-stack systems. When you need to ensure correctness across microservices, validate data pipelines, test ML models, verify GIS computations, and maintain quality in air-gapped environments—this guide provides the complete testing framework. ## Introduction diff --git a/docs/best-practices/testing/index.md b/docs/best-practices/testing/index.md index 255f744..eee1c94 100644 --- a/docs/best-practices/testing/index.md +++ b/docs/best-practices/testing/index.md @@ -1,6 +1,5 @@ # Testing Best Practices -**Objective**: Master production-grade testing strategies for distributed, multi-stack systems. When you need to ensure correctness across microservices, validate data pipelines, test ML models, verify GIS computations, and maintain quality in air-gapped environments—these testing best practices become your foundation. This collection provides comprehensive guides for unit testing, integration testing, end-to-end testing, contract testing, and QA automation. Each guide includes patterns, examples, and real-world implementation strategies. diff --git a/docs/deep-dives/index.md b/docs/deep-dives/index.md index 48109ba..5f66a4c 100644 --- a/docs/deep-dives/index.md +++ b/docs/deep-dives/index.md @@ -9,7 +9,7 @@ Some questions cannot be answered with a checklist or a firmware snippet. They d Deep Dives are analytical and comparative — the goal is understanding, not implementation. Each one examines a real tension in systems design, takes a position, and explains the reasoning. They are written for engineers who have already read the tutorials and want to think harder about what they're building. -The intellectual principles behind this section are articulated in the [Philosophy of the Site](../philosophy.md). For structured entry points by problem domain, see the [Architectural Compass](../start-here-architectural-paths.md). For curated reading sequences, see [Reading Tracks](../reading-tracks.md). Decision frameworks extracted from all essays are indexed at [Decision Frameworks](../decision-frameworks.md). +The principles behind these essays are in [Systems engineering principles](../philosophy.md). For structured entry points by problem domain, see the [Architectural Compass](../start-here-architectural-paths.md). For curated reading sequences, see [Reading Tracks](../reading-tracks.md). Decision frameworks extracted from all essays are indexed at [Decision Frameworks](../decision-frameworks.md). ## Available Deep Dives diff --git a/docs/deep-dives/mqtt-vs-http-iot.md b/docs/deep-dives/mqtt-vs-http-iot.md index abc58a1..ebe0163 100644 --- a/docs/deep-dives/mqtt-vs-http-iot.md +++ b/docs/deep-dives/mqtt-vs-http-iot.md @@ -186,5 +186,5 @@ The MQTT persistent connection model means a compromised device maintains an aut **The practical answer for most production IoT systems is not a binary choice**: cloud IoT platforms (AWS IoT Core, Azure IoT Hub, Google Cloud IoT) accept both MQTT and HTTP from devices and route messages to the same backend topics/streams. Devices can use MQTT when frequently connected and HTTP when waking from deep sleep, with the same backend consuming both. !!! tip "See also" - - [ESP32 Section Architecture](../adr/0005-esp32-section-architecture.md) — the ADR explaining how this site's ESP32 content is structured - - [ESP32 Tutorials](../tutorials/embedded/index.md) — implementation examples using MQTT with an ESP32 + - [ESP32 tutorials](../tutorials/embedded/index.md) — implementation examples using MQTT with an ESP32 + - [ESP32 best practices](../best-practices/esp32/index.md) — durable patterns for embedded MQTT clients diff --git a/docs/diagrams/style-guide.md b/docs/diagrams/style-guide.md deleted file mode 100644 index 80f6f23..0000000 --- a/docs/diagrams/style-guide.md +++ /dev/null @@ -1,232 +0,0 @@ ---- -tags: - - diagrams - - style-guide - - reference ---- - -# Diagram Style Guide - -This site uses **Mermaid** as the standard for architecture, flow, state, and sequence diagrams. SVG remains available as an exception for geospatial illustrations, high-resolution figures, and cases where Mermaid's layout engine is insufficient. - -This guide defines when to use each diagram type, formatting conventions, accessibility requirements, and a reusable snippet library. - ---- - -## When to Use Mermaid vs SVG - -| Use Mermaid for | Use SVG for | -|---|---| -| Architecture and component diagrams | Geospatial maps and spatial illustrations | -| Pipeline and data flow diagrams | High-resolution design artifacts | -| Comparison diagrams (monolith vs microservices) | Diagrams requiring precise visual positioning | -| Request/response sequence flows | Custom icons or branding elements | -| State machines (embedded firmware, auth flows) | Cases where Mermaid layout is unacceptable | -| Decision trees | — | - -**Default**: if in doubt, use Mermaid. Its text-based format is version-controlled, diffable, and editable by any contributor without tooling. - ---- - -## Diagram Types by Content - -### Flowchart — Architecture and Pipeline Diagrams - -Use `flowchart LR` (left-to-right) for most architecture and pipeline diagrams. Use `flowchart TD` (top-down) for hierarchical structures. - -``` -flowchart LR - Source --> Transform --> Sink -``` - -Best for: data pipelines, control plane / data plane separation, service topology, deployment architecture. - -### Sequence Diagram — Request and Event Flows - -Use `sequenceDiagram` for depicting interactions between services or components over time — HTTP requests, event bus publishing, auth flows. - -``` -sequenceDiagram - Client->>API: POST /request - API->>Queue: publish event - Queue->>Worker: deliver event - Worker->>API: callback result -``` - -Best for: API call flows, message broker interactions, authentication sequences, streaming event propagation. - -### State Diagram — Embedded Firmware and Lifecycle States - -Use `stateDiagram-v2` for firmware state machines, lifecycle management, and any system with discrete, well-defined states and transitions. - -``` -stateDiagram-v2 - [*] --> Idle - Idle --> Active: sensor trigger - Active --> Idle: timeout - Active --> Sleep: low battery - Sleep --> Idle: wake timer -``` - -Best for: ESP32 firmware state machines, pipeline job state transitions, deployment lifecycle stages. - ---- - -## Formatting Conventions - -**Node labels**: keep node text short — 1–4 words. Use abbreviations if needed. Long labels wrap awkwardly and reduce diagram readability. - -**Orientation**: prefer `LR` (left-to-right) for architecture and pipeline diagrams. Reserve `TD` (top-down) for hierarchies (DNS trees, org charts, decision trees). - -**Subgraphs for boundaries**: use `subgraph` blocks to show control plane / data plane separation, organizational boundaries, network zones, or logical groupings. - -``` -flowchart LR - subgraph Control Plane - Scheduler - Registry - end - subgraph Data Plane - Worker1 - Worker2 - end - Scheduler --> Worker1 - Scheduler --> Worker2 -``` - -**Complexity limit**: keep diagrams under approximately 15 nodes. If a diagram exceeds this, split it into two diagrams: one showing the high-level topology, one showing a specific subsystem in detail. - -**Consistent arrow semantics**: use `-->` for data flow, `--->` for async or eventual flow (when Mermaid supports it via labels), and `--label-->` for labeled edges when the relationship needs clarification. - ---- - -## Accessibility - -- **Avoid tiny text**: Mermaid auto-sizes nodes to their labels. Short labels produce readable diagrams; very long labels produce tiny text. Keep labels concise. -- **Avoid deeply nested subgraphs**: more than 2 levels of nesting becomes visually ambiguous. Prefer flat diagrams with clear labels over deep hierarchies. -- **Split complex diagrams**: a reader who cannot parse a diagram in 10 seconds will skip it. If the diagram requires 20+ nodes to represent the concept, split it into a sequence of simpler diagrams. -- **Light and dark mode**: MkDocs Material renders Mermaid with theme-aware colors. Do not hardcode fill colors — let the theme handle node styling for consistent light/dark rendering. - ---- - -## Mermaid Snippet Library - -### 1. Control Plane vs Data Plane - -```mermaid -flowchart LR - subgraph Control["Control Plane (governs)"] - direction TB - Catalog["Data Catalog"] - Schema["Schema Registry"] - Access["Access Policies"] - Lineage["Lineage Graph"] - end - subgraph Data["Data Plane (flows)"] - direction TB - Files["Parquet Files"] - Events["Kafka Events"] - Tables["Delta Tables"] - Models["Model Outputs"] - end - Catalog -.governs.-> Files - Schema -.enforces.-> Events - Access -.controls.-> Tables - Lineage -.tracks.-> Models -``` - -### 2. Pipeline with Stages - -```mermaid -flowchart LR - Raw["Raw Source\n(S3 / Kafka)"] - Ingest["Ingest\n(schema validate)"] - Transform["Transform\n(dbt / Spark)"] - Serve["Serve\n(warehouse / API)"] - Monitor["Monitor\n(freshness · quality)"] - - Raw --> Ingest --> Transform --> Serve - Monitor -.observes.-> Ingest - Monitor -.observes.-> Transform - Monitor -.observes.-> Serve -``` - -### 3. Monolith vs Microservices Comparison - -```mermaid -flowchart LR - subgraph Monolith["Monolith"] - direction TB - M_UI["UI"] - M_API["API Layer"] - M_BL["Business Logic"] - M_DB["Shared DB"] - M_UI --> M_API --> M_BL --> M_DB - end - subgraph Micro["Microservices"] - direction TB - S_GW["API Gateway"] - S_A["Service A\n+ DB A"] - S_B["Service B\n+ DB B"] - S_C["Service C\n+ DB C"] - S_GW --> S_A - S_GW --> S_B - S_GW --> S_C - end -``` - -### 4. Broker-Based Messaging (MQTT) - -```mermaid -flowchart LR - Sensor["ESP32 Sensor"] - Broker["MQTT Broker\n(Mosquitto)"] - Sub1["Subscriber\n(Home Assistant)"] - Sub2["Subscriber\n(Data Logger)"] - Sub3["Subscriber\n(Alert Engine)"] - - Sensor -->|"publish topic/temp"| Broker - Broker -->|"deliver"| Sub1 - Broker -->|"deliver"| Sub2 - Broker -->|"deliver"| Sub3 -``` - -### 5. Training vs Serving ML Pipeline - -```mermaid -flowchart LR - subgraph Training["Training Pipeline (offline)"] - direction LR - RawData["Raw Data"] - FeatEng["Feature Engineering\n(batch)"] - Train["Model Training\n(GPU cluster)"] - Registry["Model Registry"] - RawData --> FeatEng --> Train --> Registry - end - subgraph Serving["Serving Pipeline (online)"] - direction LR - Request["Inference Request"] - FeatStore["Feature Store\n(online)"] - Infer["Model Server"] - Response["Prediction Response"] - Request --> FeatStore --> Infer --> Response - end - Registry -.deploy model.-> Infer -``` - ---- - -## Authoring a New Diagram - -1. Choose the diagram type from the table above. -2. Keep nodes to 15 or fewer. -3. Use subgraphs to show system boundaries. -4. Label edges when the relationship is not obvious from the node names alone. -5. Test that the diagram renders in the MkDocs dev server (`mkdocs serve`) before committing. -6. If the diagram is too complex, split it before submitting. - ---- - -!!! tip "See also" - - [SVG Workflow Generation Best Practices](../best-practices/diagrams/svg-workflow-generation.md) — the full Mermaid-first, artifact-driven workflow for producing committed SVG diagrams - - [Mermaid → SVG Workflow Pipeline (Tutorial)](../tutorials/diagrams/mermaid-to-svg-workflow-pipeline.md) — step-by-step guide to running the render pipeline diff --git a/docs/documentation.md b/docs/documentation.md index 04d1fa4..0fe579d 100644 --- a/docs/documentation.md +++ b/docs/documentation.md @@ -7,7 +7,6 @@ Technical notes, mostly on geospatial systems and data infrastructure. Some page | Patterns, trade-offs, governance | [Best Practices](best-practices/index.md) | | Step-by-step builds | [Tutorials](tutorials/index.md) | | Longer essays | [Deep Dives](deep-dives/index.md) | -| Decision records | [ADRs](adr/index.md) | | Problem-oriented reading paths | [Architectural compass](start-here-architectural-paths.md) | | Creative / odd builds | [Just for Fun](tutorials/just-for-fun/index.md) | diff --git a/docs/getting-started.md b/docs/getting-started.md index 86b6ae5..069258d 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -1,136 +1,11 @@ -# Contact & Collaboration - -## Get In Touch - -Reach out about geospatial data, pipelines, or open-source work listed on [Projects](projects/index.md). - -## Contact Information +# Contact **Joshua N. Grant** -Geospatial Systems Architect -Oak Ridge National Laboratory - -### Direct Contact -- **Email**: [jngrant@live.com](mailto:jngrant@live.com) -- **Phone**: +1 (865) 803 3495 -- **Location**: Knoxville, TN 37922, United States - -### Professional Networks -- **LinkedIn**: [linkedin.com/in/joshuanagrant](https://linkedin.com/in/joshuanagrant) -- **GitHub**: [github.com/sempervent](https://github.com/sempervent) -- **Website**: [notjustadatum.blogspot.com](https://notjustadatum.blogspot.com) - -## Areas of Expertise - -### Geospatial Systems Architecture -- GeoParquet data warehouse design -- Raster pipeline development -- IoT/Kafka/TimescaleDB geospatial tracking -- Decision support tool architecture - -### Data Engineering -- ETL pipeline modernization (cron → Kubernetes/Airflow) -- Real-time data processing with Kafka -- Cloud infrastructure optimization -- CI/CD pipeline development - -### Cloud Infrastructure -- AWS, GCP, Azure platform expertise -- Container orchestration (Docker, Kubernetes) -- Serverless architecture patterns -- Microservices development - -### Full-Stack Development -- Python, R, JavaScript development -- React, Vue, Flask, FastAPI frameworks -- Database design (PostgreSQL, MongoDB, TimescaleDB) -- API development and documentation - -## Collaboration Opportunities - -### Consulting Services -- **Technical Architecture Review**: System design and optimization -- **Data Pipeline Modernization**: Legacy system upgrades -- **Cloud Migration Strategy**: On-premises to cloud transitions -- **Performance Optimization**: Database and application tuning - -### Speaking & Training -- **Technical Presentations**: Geospatial data architecture -- **Workshop Facilitation**: Data engineering best practices -- **Team Training**: Cloud infrastructure and DevOps -- **Mentoring**: Career development in data science and engineering - -### Open Source Contributions -- **Code Reviews**: Technical feedback and improvements -- **Project Collaboration**: Joint development initiatives -- **Documentation**: Technical writing and knowledge sharing -- **Community Building**: Fostering technical communities - -## Project Collaboration - -### Research & Development -- **Geospatial Data Science**: Advanced spatial analysis -- **IoT Data Processing**: Real-time sensor data handling -- **Machine Learning**: Model development and deployment -- **Blockchain Applications**: Decentralized systems development - -### Industry Applications -- **Environmental Monitoring**: Climate and ecosystem data -- **Urban Planning**: Smart city data infrastructure -- **Risk Assessment**: Geospatial risk modeling -- **Supply Chain**: Logistics and tracking systems - -## Availability - -### Current Commitments -- **Primary Role**: Geospatial Systems Architect at Oak Ridge National Laboratory -- **Research Focus**: Advanced geospatial data warehouse architectures -- **Active Projects**: Real-time IoT tracking systems - -### Collaboration Preferences -- **Remote Work**: Fully remote collaboration capabilities -- **Flexible Scheduling**: Accommodating different time zones -- **Long-term Projects**: Sustained technical partnerships -- **Quick Consultations**: Rapid technical assessments - -## Response Times - -- **Email**: Typically within 24-48 hours -- **LinkedIn**: Professional inquiries within 48 hours -- **GitHub**: Technical discussions as needed -- **Phone**: By appointment for detailed discussions - -## Next Steps - -1. **Review Portfolio**: Check out my [Projects](projects/index.md) for technical examples -2. **Technical Background**: Read my [Professional Profile](about.md) for detailed experience -3. **Best Practices**: Explore the [Best Practices](best-practices/index.md) section for proven methodologies -4. **Hands-on Tutorials**: Follow the [Tutorials](tutorials/index.md) for step-by-step implementations -5. **Reach Out**: Use any of the contact methods above to start a conversation - -## Available Resources - -### Best Practices -- **[Python Package Development](best-practices/python/python-package.md)** - Modern Python packaging with `uv` -- **[Docker & Compose](best-practices/docker-infrastructure/docker-and-compose.md)** - Production containerization -- **[Rust Development Environment](best-practices/rust/rust-dev-environment.md)** - Senior-level Rust setup -- **[Geospatial Data Engineering](best-practices/database-data/geospatial-data-engineering.md)** - Spatial data management -- **[AWS Serverless Geospatial](best-practices/database-data/aws-serverless-geospatial.md)** - Serverless spatial processing -- **[ETL Pipeline Design](best-practices/database-data/etl-pipeline-design.md)** - Robust data pipelines -- **[FastAPI Geospatial](best-practices/python/fastapi-geospatial.md)** - High-performance spatial APIs -- **[Web Performance Optimization](best-practices/python/web-performance-optimization.md)** - Core Web Vitals & SEO - -### Step-by-Step Tutorials -- **[PostGIS Geometry Indexing](tutorials/database-data-engineering/postgis-geometry-indexing.md)** - Spatial database optimization -- **[PostGIS Raster Indexing](tutorials/database-data-engineering/postgis-raster-indexing.md)** - Raster data management -- **[Raster–Vector Workflows](tutorials/database-data-engineering/postgis-raster-vector-workflows.md)** - Hybrid spatial analysis -- **[parquet_s3_fdw with Multiple Backends](tutorials/database-data-engineering/parquet-s3-fdw.md)** - Object store integration -- **[GeoParquet with Polars](tutorials/database-data-engineering/geoparquet-with-polars.md)** - High-performance spatial analytics -- **[Kafka + TimescaleDB IoT Streaming](tutorials/database-data-engineering/kafka-timescaledb-iot.md)** - Real-time data pipelines -- **[R Shiny Geospatial Apps](tutorials/python-development/r-shiny-geoapp.md)** - Interactive spatial applications -- **[Monitoring with Grafana & Prometheus](tutorials/quick-start/monitoring-with-grafana-prometheus.md)** - Observability stack - -### Core Documentation -- [Professional Profile](about.md) - Detailed background and experience -- [Projects Portfolio](projects/index.md) - Technical implementations and case studies -- [Technical Documentation](documentation.md) - Overview of methodologies and approaches +Geospatial Systems Architect, Oak Ridge National Laboratory + +- **Email:** [jngrant@live.com](mailto:jngrant@live.com) +- **Phone:** +1 (865) 803-3495 +- **Location:** Knoxville, TN, United States +- **GitHub:** [github.com/sempervent](https://github.com/sempervent) +- **LinkedIn:** [linkedin.com/in/joshuanagrant](https://linkedin.com/in/joshuanagrant) +- **Blog:** [notjustadatum.blogspot.com](https://notjustadatum.blogspot.com) diff --git a/docs/index.md b/docs/index.md index 83d5927..56d1eb3 100644 --- a/docs/index.md +++ b/docs/index.md @@ -24,7 +24,7 @@ Joshua N. Grant — geospatial systems architect. Most of my time goes to spatia ## Writing -Notes from production geospatial work, plus tutorials and longer essays. [Best practices](best-practices/index.md) for patterns and trade-offs; [tutorials](tutorials/index.md) for hands-on builds; [deep dives](deep-dives/index.md) and [ADRs](adr/index.md) when I had more to say. +Notes from production geospatial work, plus tutorials and longer essays. [Best practices](best-practices/index.md) for patterns and trade-offs; [tutorials](tutorials/index.md) for hands-on builds; [deep dives](deep-dives/index.md) for longer essays. [Architectural compass](start-here-architectural-paths.md) @@ -47,4 +47,4 @@ Notes from production geospatial work, plus tutorials and longer essays. [Best p [jngrant@live.com](mailto:jngrant@live.com) · [@sempervent](https://github.com/sempervent) · [LinkedIn](https://linkedin.com/in/joshuanagrant) · [Not Just a Datum](https://notjustadatum.blogspot.com) -Personal site — not an ORNL publication. [Profile](about.md) · [Contact form & details](getting-started.md) +Personal site — not an ORNL publication. [About](about.md) · [Contact](getting-started.md) diff --git a/docs/maintainers/content-taxonomy-audit-2026-09-30.md b/docs/maintainers/content-taxonomy-audit-2026-09-30.md deleted file mode 100644 index 9716f2b..0000000 --- a/docs/maintainers/content-taxonomy-audit-2026-09-30.md +++ /dev/null @@ -1,43 +0,0 @@ -# Content taxonomy audit (2026-09-30) - -Scope: Best Practices vs Tutorials vs Just for Fun under **Writing**, after the portfolio IA pass. - -## Category contracts (unchanged) - -| Category | Question it answers | -| --- | --- | -| **Best Practices** | What should practitioners understand about design, trade-offs, failure modes, and operations? | -| **Tutorials** | How do I build, configure, or run this specific thing? | -| **Just for Fun** | What happens when curiosity drives the stack — art, games, music, odd integrations? | - -**Lab** top-level nav was removed: it duplicated **Projects** (creative repos) and **Just for Fun** (creative tutorials). Redirect: `lab/index.md` → Just for Fun index. - -## Moves performed - -| Current path | From | To | Reason | Redirect | -| --- | --- | --- | --- | --- | -| `tutorials/just-for-fun/js-glitch-observatory.md` | Tutorials → Python Development | Tutorials → Just for Fun | Browser entropy/art piece; not a Python dev guide | `tutorials/python-development/js-glitch-observatory.md` → new path | - -## Reviewed; kept in place - -| Path | Category | Notes | -| --- | --- | --- | -| `tutorials/just-for-fun/managing-people-software-dev.md` | Just for Fun (nav) | Leadership essay with tutorial shape; thematically odd but stable URL and useful cross-link target — leave unless a dedicated “Engineering leadership” section appears later | -| `best-practices/creative-fun/*` | Best Practices | Small operational patterns (Celery, LaTeX, time hygiene) — name is cute, content is pattern-oriented; no move | -| `tutorials/best-practices-integration/*` | (mostly unlisted in nav) | Long integration walkthroughs; remain under Tutorials tree for now — audit did not mass-move | -| `deep-dives/*` | Deep Dives (under Writing) | Essay-style; distinct from Best Practices indexes — no change | - -## Writing / Projects / Just for Fun hierarchy (decision) - -**Option A adopted:** - -- **Projects** — repositories and docs sites (registry-driven). -- **Writing** — Best Practices, Tutorials (including **Just for Fun** nested under Tutorials). -- **About** — profile and contact. - -No separate **Lab** tab. - -## Follow-up (optional, low priority) - -- Consider moving `managing-people-software-dev.md` under Best Practices → architecture/leadership if the section grows. -- Periodically spot-check new tutorials landing in `python-development/` for creative misfiles. diff --git a/docs/maintainers/portfolio-recon-2026-09-30.md b/docs/maintainers/portfolio-recon-2026-09-30.md deleted file mode 100644 index d902d7d..0000000 --- a/docs/maintainers/portfolio-recon-2026-09-30.md +++ /dev/null @@ -1,97 +0,0 @@ -# Portfolio & site reconnaissance (2026-09-30) - -Evidence sources: this repository (`main`), live site `https://sempervent.github.io/`, -GitHub API (`sempervent/*` public repos), and HTTP checks against org GitHub Pages paths. - -## Current site architecture (before restructure) - -| Layer | Behavior | -| --- | --- | -| Stack | MkDocs 1.x + Material 9.x, GitHub Actions → GitHub Pages (`deploy-mkdocs.yml`) | -| Home (`docs/index.md`) | Hero + **Best Practices / Tutorials / Projects / Just for Fun** cards; deep “Start Here” and featured articles; stale **What I'm Working On** and **Latest Updates** | -| Nav (`mkdocs.yml`) | Many top-level tabs: Home, What's New, Tags, Profile, Projects, Technical Documentation, Doctrine, ADRs, Best Practices, Deep Dives, Tutorials, Contact | -| Projects | Single hand-maintained `docs/projects.md` — foregrounds BlackLake, ORNL OpenSAMPL, and legacy web apps | -| Freshness | Manual lists on homepage; `whats-new.md` is explicitly curated (ADR-0001) | -| Project metadata | Duplicated prose in `projects.md` / homepage; **no canonical registry** | -| Plugins installed | `git-revision-date-localized`, `tags`, `minify`; `mkdocs-redirects` / `macros` in `requirements.txt` but **not wired** in `mkdocs.yml` | - -## Stale or misleading sections - -- Homepage **Featured** projects: Final Fantasy Football, Where I've Been, This Is A Casino, DCRS — not reflected in recent `sempervent` GitHub activity; several lack repo links in `projects.md`. -- **What I'm Working On** — generic doc links, not tied to active repositories (PARQONAUT, `dots`, NUMBRANE). -- **About This Site** — “Everything here is battle-tested” / “production-ready configurations” overstates mixed content (experiments, tutorials, reference). -- **BlackLake** — `projects.md` links “Live Demo” to `s3-rust-data-portal` Pages only; separate `blacklake` Python repo also publishes docs at `/blacklake/`. -- **Footer** — presents ORNL affiliation without personal-site disclaimer. - -## Verified live GitHub Pages (HTTP 200) - -Org site base: `https://sempervent.github.io/` - -| Path | Repo | Notes | -| --- | --- | --- | -| `/` | `sempervent.github.io` | Main portfolio / docs | -| `/PARQONAUT/` | `PARQONAUT` | Homepage URL set on repo | -| `/dots/` | `dots` | Dotfiles docs | -| `/music-rig/` | `music-rig` | Studio setup docs | -| `/blacklake/` | `blacklake` | Python Blacklake docs | -| `/s3-rust-data-portal/` | `s3-rust-data-portal` | Rust portal docs (README also branded Blacklake) | -| `/gi/` | `gi` | | -| `/wildfire-smoke-risk-correlator/` | `wildfire-smoke-risk-correlator` | | -| `/agent-llm-wiki-matrix/` | `agent-llm-wiki-matrix` | | -| `/postgres-query-autopsy-tool/` | `postgres-query-autopsy-tool` | | -| `/smart-farm-wiki/` | `smart-farm-wiki` | | -| `/llm-wiki-template/` | `llm-wiki-template` | | -| `/generative-midi-workbench/` | `generative-midi-workbench` | | - -External: [OpenSAMPL docs](https://ornl.github.io/OpenSAMPL/) (`ORNL/OpenSAMPL`). - -## Pages enabled but not serving (HTTP 404 on org URL) - -GitHub Pages API reports a site; path returned 404 on 2026-09-30: - -| Repo | Configured URL | -| --- | --- | -| `mqtt-comparison` | `https://sempervent.github.io/mqtt-comparison/` | -| `embers-of-the-earth` | `https://sempervent.github.io/embers-of-the-earth/` | -| `colony` | `https://sempervent.github.io/colony/` | -| `universe` | `https://sempervent.github.io/universe/` | - -Omitted from the public “documentation sites” directory until deploys succeed. - -## Orphaned / unlinked live Pages - -Live sites above were **not** linked from the main portfolio prior to this work (except BlackLake via wrong repo URL). `PARQONAUT`, `dots`, wikis, and tooling docs were absent from `projects.md`. - -## Recently active public repos (portfolio-relevant) - -By `updatedAt` on 2026-09-30: `numbrane`, `dots`, `PARQONAUT`, `paraclete`, `music-rig`, `cosmic-architect`, `wildfire-smoke-risk-correlator`, agent/wiki tooling, `postgres-query-autopsy-tool`. - -## Recommended hierarchy - -1. **Person** — hero + professional scope (geospatial, data, distributed systems, tooling). -2. **Current work** — registry-driven cards: PARQONAUT, dots, NUMBRANE, Paraclete, Cosmic Architect. -3. **Proof** — featured engineering (Blacklake lineages, smoke correlator, autopsy tool). -4. **Technical writing** — Doctrine / Best Practices / Deep Dives / Tutorials (unchanged URLs, grouped under **Writing** tab). -5. **Lab** — creative, games, MIDI, Pi experiments. -6. **Deep docs** — existing taxonomy preserved. - -## BlackLake lineage (evidence-based) - -| Repo | Implementation | Docs URL | Last push (API) | -| --- | --- | --- | --- | -| `sempervent/blacklake` | Python; git-like S3 dataset store + semantic metadata | `/blacklake/` | 2025-10-09 | -| `sempervent/s3-rust-data-portal` | Rust/Axum ML artifact portal; README title “Blacklake” | `/s3-rust-data-portal/` | 2025-12-16 | - -Both are public, both publish MkDocs sites under the org Pages namespace. They appear to be **parallel lineages** sharing a product name rather than a simple rename; this site lists them separately and avoids implying a single merged codebase. - -## Deployment vs repository - -- CI builds with `mkdocs build --clean` (no `--strict` on `main` deploy); PR preview uses `--strict`. -- No pre-build project generator existed before this change. - -## Implementation follow-up (this branch) - -- Canonical registry: `data/projects.yaml` -- Generator: `scripts/generate_portfolio.py` -- Validation: `scripts/validate_projects.py` -- Redirect: legacy `projects.md` → `projects/index.md` diff --git a/docs/philosophy.md b/docs/philosophy.md index 5d27965..931bbc9 100644 --- a/docs/philosophy.md +++ b/docs/philosophy.md @@ -1,14 +1,12 @@ --- tags: - - doctrine + - guides - philosophy --- -# Philosophy of the Site +# Systems engineering principles -This site is not a collection of documentation. It is a structured argument about how to build systems well — at the level of architecture, economics, and organizational discipline. - -The analytical essays, best practices, and tutorials published here are expressions of a consistent intellectual position. That position has five principles. +These are the principles I use when judging architecture, platforms, and organizational trade-offs. The [deep dives](deep-dives/index.md), [best practices](best-practices/index.md), and [tutorials](tutorials/index.md) apply them to specific domains. --- @@ -16,7 +14,7 @@ The analytical essays, best practices, and tutorials published here are expressi Every decade produces a new class of technologies that promise to eliminate the problems of the previous decade. Microservices promised to fix the monolith. Kubernetes promised to fix microservice deployment. Serverless promised to fix Kubernetes overhead. The promise is always partially true and the cost is always underestimated. -The appropriate response to a new technology is not adoption and not rejection, but analysis: what problem does this solve, what does it cost, and do I have that problem at that cost? The essays on this site apply this question consistently. The answer is frequently "not yet" or "not for this use case." +The appropriate response to a new technology is not adoption and not rejection, but analysis: what problem does this solve, what does it cost, and do I have that problem at that cost? The answer is frequently "not yet" or "not for this use case." Restraint is not conservatism. It is the discipline of not paying complexity costs for capabilities you do not need. @@ -28,7 +26,7 @@ Technical decisions have economic consequences that outlast the enthusiasm that Economics here means the full accounting: engineering time, infrastructure cost, opportunity cost, organizational learning overhead, and the switching cost of the decision being wrong. A technology that looks cheap on a benchmark but expensive in operation has been incorrectly evaluated. -The essays on this site consistently ask: what does this actually cost, over what time horizon, in organizations of what size and maturity? The answer to that question is more durable than any benchmark. +Ask what a choice actually costs, over what time horizon, in organizations of what size and maturity. That answer is more durable than any benchmark. --- @@ -36,7 +34,7 @@ The essays on this site consistently ask: what does this actually cost, over wha Abstractions are valuable precisely because they hide detail. They become a liability when the hidden detail becomes a failure mode. An engineer who does not understand what happens when a serverless function cold starts, what etcd does when disk latency spikes, or what "eventual consistency" means for their read pattern has adopted an abstraction without understanding its failure envelope. -Determinism — the ability to predict and reason about system behavior under load, failure, and edge conditions — requires understanding the implementation that abstractions hide. This site regularly descends below the abstraction layer: into storage physics, into network protocol semantics, into scheduling mechanics. This descent is not academic. It is the foundation of reliable engineering judgment. +Determinism — the ability to predict and reason about system behavior under load, failure, and edge conditions — requires understanding the implementation that abstractions hide. Reliable engineering judgment often requires descending below the abstraction layer: into storage physics, network protocol semantics, and scheduling mechanics. --- @@ -44,26 +42,24 @@ Determinism — the ability to predict and reason about system behavior under lo Data systems, infrastructure, and organizational processes that grow without governance accumulate entropy. Data lakes become swamps. Microservice deployments become dependency graphs no one understands. Kubernetes clusters accumulate Helm releases that no one is responsible for. Metadata becomes stale. Pipelines multiply without ownership. -The essays on this site treat governance not as bureaucracy but as the structural discipline that makes growth sustainable. Schema contracts, ownership models, metadata enforcement, and architectural decision records are not overhead — they are the mechanism by which a system remains understandable as it grows. The cost of governance is always lower than the cost of rearchitecting from chaos. +Governance is not bureaucracy; it is the structural discipline that makes growth sustainable. Schema contracts, ownership models, metadata enforcement, and recorded architectural decisions are mechanisms that keep a system understandable as it grows. The cost of governance is usually lower than the cost of rearchitecting from chaos. --- ## Discipline Over Novelty -The most durable engineering decisions are conservative: they use the simplest technology that solves the problem, they prefer well-understood failure modes over novel ones, and they resist the organizational pressure to adopt new tools before the existing tools have been exhausted. +The most durable engineering decisions are conservative: they use the simplest technology that solves the problem, they prefer well-understood failure modes over novel ones, and they resist organizational pressure to adopt new tools before existing tools have been exhausted. -The essays on this site are not hostile to new technologies. DuckDB, H3, Apache Iceberg, and OpenTelemetry appear throughout because they represent genuine improvements over their predecessors in specific domains. But they appear in analytical context: what problem they solve, what they cost, and when the older approach remains correct. +That is not hostility to new technology. DuckDB, H3, Apache Iceberg, and OpenTelemetry appear throughout these notes because they represent genuine improvements in specific domains — always with context on what problem they solve, what they cost, and when the older approach remains correct. Novelty is a feature when it solves a real problem. It is a liability when it is pursued for its own sake. --- -## Reading This Site - -These principles are expressed across every section of this site: +## Related reading -- [Deep Dives](deep-dives/index.md) — long-form essays applying these principles to specific architectural decisions -- [Reading Tracks](reading-tracks.md) — curated sequences for readers entering specific domains -- [Decision Frameworks](decision-frameworks.md) — the structured decision tools extracted from the essays -- [Anti-Patterns](anti-patterns.md) — the recurring mistakes these principles are designed to prevent -- [Systems Glossary](systems-glossary.md) — the vocabulary shared across the analytical essays +- [Deep Dives](deep-dives/index.md) — long-form essays on specific architectural decisions +- [Reading Tracks](reading-tracks.md) — curated sequences by domain +- [Decision Frameworks](decision-frameworks.md) — structured decision tools extracted from the essays +- [Anti-Patterns](anti-patterns.md) — recurring mistakes these principles are meant to prevent +- [Systems Glossary](systems-glossary.md) — shared vocabulary across the essays diff --git a/docs/start-here-architectural-paths.md b/docs/start-here-architectural-paths.md index 79c6ba4..8e67961 100644 --- a/docs/start-here-architectural-paths.md +++ b/docs/start-here-architectural-paths.md @@ -94,4 +94,4 @@ You are selecting between Kubernetes and simpler deployment models, evaluating s If you encounter terms used across multiple essays without definition, the [Systems Thinking Glossary](systems-glossary.md) defines the shared vocabulary: control plane, data plane, abstraction debt, metadata debt, data gravity, blast radius, eventual consistency, schema contract, and operational entropy. -For the intellectual principles behind the analytical positions taken across this site, read the [Philosophy of the Site](philosophy.md). +For the engineering principles behind these paths, see [Systems engineering principles](philosophy.md). diff --git a/docs/tags.md b/docs/tags.md deleted file mode 100644 index dec39a5..0000000 --- a/docs/tags.md +++ /dev/null @@ -1,5 +0,0 @@ -# Tags - -Browse all content by topic tag. - -[TAGS] diff --git a/docs/tutorials/best-practices-integration/index.md b/docs/tutorials/best-practices-integration/index.md index 5f48bda..003f5fd 100644 --- a/docs/tutorials/best-practices-integration/index.md +++ b/docs/tutorials/best-practices-integration/index.md @@ -1,315 +1,10 @@ -# Best Practices Integration Tutorials +# Archived: Best Practices Integration tutorials -**Objective**: Master complex technical implementations that combine multiple best practices into cohesive, production-ready systems. When you need to see how best practices work together, when you want integrated solutions, when you need real-world examples—these tutorials become your weapon of choice. +These multi-topic walkthroughs are **archived** while the writing corpus is cleaned up. They read like generated course composites and are not maintained as live tutorials. -This collection provides comprehensive, hands-on tutorials that demonstrate how multiple best practices integrate to solve real-world problems. Each tutorial combines 3-5 related best practices into a complete, working system. +Use instead: -## Overview - -These tutorials go beyond individual best practices to show how they work together in production systems. Each tutorial includes: - -- **Complete working code** across multiple languages (Python, Go, Rust) -- **Integration patterns** showing how best practices complement each other -- **Production-ready configurations** for Docker, Kubernetes, and infrastructure -- **Observability** built in from day one -- **Testing strategies** including chaos engineering and quality validation - -## Tutorials - -### 1. Event-Driven Microservices with Observability and Data Governance - -**[Complete Tutorial](event-driven-microservices-observability-data-governance.md)** - -Build a production-ready event-driven microservices system that integrates: -- Event-driven architecture patterns -- Observability-driven development -- Data validation and contract governance -- API gateway architecture -- Service decomposition strategies - -**Technologies**: Python, Go, FastAPI, Kafka, PostgreSQL, Redis, Kong, Prometheus, Grafana, Jaeger - -**Use Cases**: Order processing systems, real-time data pipelines, microservices architectures - ---- - -### 2. Multi-Environment Deployment with Secrets and Configuration Governance - -**[Complete Tutorial](multi-environment-deployment-secrets-config-governance.md)** - -Build a secure multi-environment deployment pipeline that integrates: -- Cross-environment configuration governance -- End-to-end secrets management -- CI/CD pipelines with validation -- Configuration drift prevention -- Environment promotion workflows - -**Technologies**: Kubernetes, Helm, Ansible, SOPS, GitHub Actions, Prometheus - -**Use Cases**: Multi-environment deployments, configuration management, secrets rotation - ---- - -### 3. Data Pipeline with Quality Governance and Observability - -**[Complete Tutorial](data-pipeline-quality-governance-observability.md)** - -Build a production-ready data pipeline with comprehensive quality governance that integrates: -- Data quality SLAs and validation layers -- Data validation and contract governance -- Metadata standards and provenance tracking -- Data freshness SLA governance -- Observability-driven development - -**Technologies**: Prefect, Great Expectations, DuckDB, PostgreSQL, Parquet, Prometheus - -**Use Cases**: ETL pipelines, data quality assurance, data lineage tracking - ---- - -### 4. Resilient Microservices with Chaos Engineering and Operational Resilience - -**[Complete Tutorial](resilient-microservices-chaos-engineering.md)** - -Build a production-ready resilient microservices system that integrates: -- Chaos engineering and fault injection -- Operational resilience patterns -- System resilience (circuit breakers, retries, timeouts) -- Observability-driven development -- Service decomposition for resilience - -**Technologies**: Python, FastAPI, Chaos Mesh, Prometheus, Grafana, Kubernetes - -**Use Cases**: High-availability systems, fault-tolerant architectures, reliability validation - ---- - -### 5. Secure Polyglot System with Identity Federation and Encryption Lifecycle - -**[Complete Tutorial](secure-polyglot-identity-encryption.md)** - -Build a production-ready secure polyglot system that integrates: -- Cross-domain identity federation (OIDC, JWT, RBAC/ABAC) -- Secret supply chains and encryption lifecycle management -- End-to-end secrets management -- Secure-by-design lifecycle architecture -- Multi-tenant isolation and sandboxing - -**Technologies**: Python, Go, Rust, FastAPI, Hashicorp Vault, OIDC, Kubernetes - -**Use Cases**: Multi-tenant SaaS, secure APIs, identity federation, encryption management - ---- - -### 6. Cost-Optimized ML Pipeline with Capacity Planning and Data Retention - -**[Complete Tutorial](cost-optimized-ml-pipeline-capacity-retention.md)** - -Build a production-ready ML pipeline that integrates: -- Cost-aware architecture and resource-efficiency governance -- Holistic capacity planning and workload modeling -- Data retention, archival strategy, and lifecycle governance -- ML systems architecture with feature stores and model serving - -**Technologies**: Python, MLflow, Prefect, DuckDB, PostgreSQL, Parquet, S3, Prometheus, Kubecost - -**Use Cases**: ML training pipelines, cost-optimized model serving, data lifecycle management - ---- - -### 7. Multi-Region Disaster Recovery with Temporal Governance and Blast Radius Reduction - -**[Complete Tutorial](multi-region-dr-temporal-blast-radius.md)** - -Build a production-ready multi-region DR system that integrates: -- Multi-region, multi-cluster disaster recovery strategies -- Temporal governance and time synchronization -- Operational risk modeling and blast radius reduction -- Release management and progressive delivery - -**Technologies**: Kubernetes, PostgreSQL, Patroni, Chrony/NTP, Prometheus, Grafana - -**Use Cases**: Multi-region deployments, disaster recovery, time-consistent systems - ---- - -### 8. Developer Experience Platform with System Taxonomy and Repository Standardization - -**[Complete Tutorial](developer-experience-taxonomy-repository.md)** - -Build a comprehensive developer experience platform that integrates: -- System-wide naming, taxonomy, and structural vocabulary governance -- Repository standardization and governance -- Cognitive load management and developer experience -- Architectural fitness functions and governance - -**Technologies**: Python, CookieCutter, pre-commit, Prometheus, Git - -**Use Cases**: Developer tooling, repository templates, DX improvement, architecture quality - ---- - -### 9. Progressive Delivery System with Release Management and Fitness Functions - -**[Complete Tutorial](progressive-delivery-release-fitness.md)** - -Build a production-ready progressive delivery system that integrates: -- Release management, change governance, and progressive delivery -- Architectural fitness functions and governance -- Operational risk modeling and blast radius reduction -- Observability-driven development - -**Technologies**: Kubernetes, Argo Rollouts, Istio, Prometheus, Grafana, Alertmanager - -**Use Cases**: Safe deployments, canary releases, blue-green deployments, quality gates - ---- - -### 10. Geospatial Data Mesh with Cost Optimization and Capacity Planning - -**[Complete Tutorial](geospatial-data-mesh-cost-capacity.md)** - -Build a production-ready geospatial data mesh that integrates: -- Data mesh architecture with domain-oriented design -- Cost-aware architecture and resource-efficiency governance -- Holistic capacity planning and workload modeling -- Data retention, archival strategy, and lifecycle governance - -**Technologies**: PostGIS, DuckDB, GeoParquet, S3, Python, GeoPandas - -**Use Cases**: Geospatial data platforms, satellite imagery processing, spatial analytics - ---- - -### 11. Polyglot Streaming Platform with SAGA, CQRS, and Multi-Cloud Portability - -**[Complete Tutorial](polyglot-streaming-saga-cqrs-multicloud.md)** - -Build a production-ready polyglot streaming platform that integrates: -- Streaming architecture patterns (SAGA, CQRS, Outbox) -- Polyglot interoperability design across Python, Go, Rust -- Multi-cloud federation and portability architecture -- API governance with backward compatibility - -**Technologies**: Python, Go, Rust, Kafka, Protocol Buffers, gRPC, Kubernetes, AWS, GCP, Azure - -**Use Cases**: Multi-cloud event-driven systems, distributed transactions, polyglot microservices - ---- - -### 12. Semantic Knowledge Graph Platform with RDF/OWL and Data Lineage - -**[Complete Tutorial](semantic-knowledge-graph-rdf-lineage.md)** - -Build a production-ready semantic knowledge graph platform that integrates: -- Semantic layer engineering with domain models -- RDF/OWL metadata automation and reasoning -- Cross-system data lineage and provenance tracking -- Metadata standards and schema governance - -**Technologies**: Python, RDFLib, OWLReady2, Neo4j, SPARQL, PostgreSQL, DuckDB - -**Use Cases**: Knowledge graphs, semantic data platforms, data lineage tracking, metadata management - ---- - -### 13. High-Performance Caching Architecture with Cache Topology - -**[Complete Tutorial](high-performance-caching-topology.md)** - -Build a production-ready multi-tier caching system that integrates: -- Cache-topology architecture with 10-tier hierarchy -- End-to-end caching strategy and performance layering -- Cost-aware architecture and resource efficiency -- Holistic capacity planning and workload modeling - -**Technologies**: Redis, NGINX, PostgreSQL, Python, Prometheus, Docker - -**Use Cases**: High-performance APIs, ETL caching, ML inference caching, GIS tile caching - ---- - -### 14. API-First Polyglot Platform with Protocol Buffers - -**[Complete Tutorial](api-first-polyglot-protobuf.md)** - -Build a production-ready API-first polyglot platform that integrates: -- API governance with backward compatibility rules -- Protocol Buffers for efficient serialization -- Polyglot interoperability design -- Service decomposition strategies - -**Technologies**: Python, Go, Rust, Protocol Buffers, gRPC, Kong, FastAPI - -**Use Cases**: Polyglot microservices, type-safe APIs, API versioning, cross-language communication - ---- - -### 15. Documentation-Driven Development with ADR Governance - -**[Complete Tutorial](documentation-driven-adr-governance.md)** - -Build a comprehensive documentation platform that integrates: -- Documentation best practices and organization -- ADR (Architecture Decision Records) governance -- Reference architecture diagrams -- Cognitive load management and developer experience - -**Technologies**: MkDocs, Mermaid, Python, Markdown, Git - -**Use Cases**: Technical documentation, architecture decision tracking, developer onboarding, system documentation - ---- - -## How to Use These Tutorials - -### Prerequisites - -Each tutorial includes specific prerequisites, but generally you'll need: -- Docker and Docker Compose -- Kubernetes cluster (local or remote) -- Programming languages: Python 3.10+, Go 1.21+, Rust 1.70+ -- Infrastructure tools: kubectl, helm, ansible -- Security tools: vault, sops, age - -### Learning Path - -1. **Start with the basics**: Ensure you understand the individual best practices referenced in each tutorial -2. **Follow step-by-step**: Each tutorial is designed to be followed sequentially -3. **Experiment**: Modify configurations and code to understand how changes affect the system -4. **Extend**: Use these tutorials as starting points for your own implementations - -### Integration Patterns - -These tutorials demonstrate common integration patterns: - -- **Observability Integration**: All tutorials show how to integrate metrics, tracing, and logging -- **Security Integration**: Security patterns are woven throughout, not bolted on -- **Quality Integration**: Data quality and validation are built into pipelines from the start -- **Resilience Integration**: Resilience patterns work together to create robust systems - -## Best Practices Covered - -These tutorials integrate best practices from: - -- **[Architecture & Design](../../best-practices/architecture-design/index.md)** -- **[Data Governance](../../best-practices/data-governance/index.md)** -- **[Operations & Monitoring](../../best-practices/operations-monitoring/index.md)** -- **[Security](../../best-practices/security/index.md)** -- **[Python Development](../../best-practices/python/index.md)** -- **[Go Development](../../best-practices/go/index.md)** -- **[Rust Development](../../best-practices/rust/index.md)** - -## Next Steps - -After completing these tutorials: - -1. **Adapt to your needs**: Modify the examples to fit your specific use cases -2. **Combine patterns**: Mix and match patterns from different tutorials -3. **Extend functionality**: Add features and capabilities beyond what's shown -4. **Share improvements**: Contribute back improvements and extensions - ---- - -*These tutorials demonstrate how multiple best practices integrate to create production-ready systems. Each tutorial includes complete code, configurations, and real-world patterns for enterprise deployment.* +- [Tutorials index](../index.md) +- [Best Practices index](../../best-practices/index.md) +Source copies remain under `docs/archive/tutorials/best-practices-integration/` in the repository for reference. diff --git a/docs/tutorials/data-science-visualization/index.md b/docs/tutorials/data-science-visualization/index.md index b719897..1c01454 100644 --- a/docs/tutorials/data-science-visualization/index.md +++ b/docs/tutorials/data-science-visualization/index.md @@ -1,6 +1,5 @@ # Data Science & Visualization Tutorials -**Objective**: Master complex data science and visualization implementations through step-by-step guides. When you need to implement specific data science technologies, when you want to follow proven patterns, when you need copy-paste runnable examples—these tutorials become your weapon of choice. ## Jupyter & Notebooks @@ -14,4 +13,3 @@ --- -*These tutorials provide the complete machinery for implementing key data science and visualization technologies and workflows. Each guide includes production-ready examples, configuration files, and best practices for enterprise deployment.* diff --git a/docs/tutorials/data-science-visualization/jupyter-notebook-best-practices-geo.md b/docs/tutorials/data-science-visualization/jupyter-notebook-best-practices-geo.md index 05797f9..f66d90d 100644 --- a/docs/tutorials/data-science-visualization/jupyter-notebook-best-practices-geo.md +++ b/docs/tutorials/data-science-visualization/jupyter-notebook-best-practices-geo.md @@ -1,6 +1,5 @@ # Jupyter Notebooks for Data Exploration — Best Practices (Geospatial Edition) -**Objective**: Master the art of building reproducible, efficient Jupyter notebooks for geospatial data exploration. Transform your notebooks from chaotic experiments into professional, maintainable analysis tools. The notebook is a lab bench: wires everywhere, chemicals hissing, truth under pressure. Keep it fast, clean, and reproducible—or it will turn on you. @@ -599,4 +598,3 @@ Jupyter notebooks are powerful tools for geospatial exploration, but they requir --- -*This tutorial provides the complete machinery for building professional geospatial notebooks. The patterns scale from exploration to production, from megabytes to terabytes.* diff --git a/docs/tutorials/data-science-visualization/latex-tikz-diagrams.md b/docs/tutorials/data-science-visualization/latex-tikz-diagrams.md index 320cfa2..4995e35 100644 --- a/docs/tutorials/data-science-visualization/latex-tikz-diagrams.md +++ b/docs/tutorials/data-science-visualization/latex-tikz-diagrams.md @@ -1,6 +1,5 @@ # Creating Beautiful Diagrams in LaTeX with TikZ -**Objective**: Master the art of creating stunning, publication-ready diagrams using LaTeX's TikZ vector graphics language. Transform your documentation from static text into living, breathing visualizations. TikZ is LaTeX's vector graphics language. It lets you produce high-quality, publication-ready diagrams inside documents. Unlike drawing tools, TikZ is precise, programmable, and beautifully typeset. It's not just for boring flowcharts—it's a canvas for elegant visualizations, fractals, and diagrams that feel alive. @@ -526,4 +525,3 @@ TikZ transforms your documentation from static text into living, breathing visua --- -*This tutorial provides the complete machinery for creating beautiful diagrams with TikZ in LaTeX. The diagrams live with your documentation, update with your code, and scale with your creativity.* diff --git a/docs/tutorials/data-science-visualization/mermaid-diagrams.md b/docs/tutorials/data-science-visualization/mermaid-diagrams.md index fa9a010..3ad30eb 100644 --- a/docs/tutorials/data-science-visualization/mermaid-diagrams.md +++ b/docs/tutorials/data-science-visualization/mermaid-diagrams.md @@ -1,6 +1,5 @@ # Creating Beautiful Diagrams with Mermaid in Markdown (MkDocs) -**Objective**: Master the art of creating dynamic, beautiful diagrams with Mermaid directly in your Markdown files, eliminating the need for static images and external diagramming tools. Mermaid is a lightweight diagramming and charting tool that turns text into beautiful diagrams. It integrates seamlessly with MkDocs (especially when using the Material for MkDocs theme) and allows you to embed diagrams directly inside your Markdown files. @@ -482,4 +481,4 @@ graph LR ``` --> -``` \ No newline at end of file +``` diff --git a/docs/tutorials/data-science-visualization/r-generative-art.md b/docs/tutorials/data-science-visualization/r-generative-art.md index b570d9d..6686c44 100644 --- a/docs/tutorials/data-science-visualization/r-generative-art.md +++ b/docs/tutorials/data-science-visualization/r-generative-art.md @@ -1,6 +1,5 @@ # Creating Beautiful Generative Art in R -**Objective**: Master generative art creation using R's powerful visualization capabilities. When you need to create algorithmic art, when you want to explore mathematical beauty, when you're building data-driven visualizations—generative art becomes your weapon of choice. R is not just for statistics—it's a powerful tool for creating beautiful, algorithmic art. This tutorial shows you how to wield R's visualization capabilities with the precision of a digital artist, covering everything from basic geometric patterns to complex mathematical art and interactive visualizations. @@ -731,4 +730,3 @@ Generative art requires understanding both mathematical patterns and aesthetic p --- -*This tutorial provides the complete machinery for generative art in R. The patterns scale from simple geometric patterns to complex mathematical art, from basic plotting to advanced interactive visualizations.* diff --git a/docs/tutorials/database-data-engineering/alembic-migrations.md b/docs/tutorials/database-data-engineering/alembic-migrations.md index 5239ad7..c04e1ca 100644 --- a/docs/tutorials/database-data-engineering/alembic-migrations.md +++ b/docs/tutorials/database-data-engineering/alembic-migrations.md @@ -1,6 +1,5 @@ # Taming Database Evolution with Alembic -**Objective**: Master Alembic migrations to manage database schema changes with surgical precision. When your database schema evolves faster than your application, when your team needs reproducible database changes, when production deployments require zero-downtime schema updates—Alembic becomes your weapon of choice. Database migrations are the bridge between your evolving application and its persistent storage. Without them, you're flying blind into production with schema changes that could break everything. This guide shows you how to wield Alembic with the precision of a seasoned backend engineer. @@ -1150,4 +1149,3 @@ Alembic migrations are the foundation of reliable database schema management. Wh --- -*This tutorial provides the complete machinery for managing database evolution with Alembic. The patterns scale from development to production, from simple schema changes to complex data migrations.* diff --git a/docs/tutorials/database-data-engineering/apache-spark-mastery.md b/docs/tutorials/database-data-engineering/apache-spark-mastery.md index f7eea55..bbe8a2b 100644 --- a/docs/tutorials/database-data-engineering/apache-spark-mastery.md +++ b/docs/tutorials/database-data-engineering/apache-spark-mastery.md @@ -1,6 +1,5 @@ # Apache Spark Mastery: Complete Guide to Distributed Data Processing -**Objective**: Master Apache Spark for distributed data processing, analytics, and machine learning. When you need to process massive datasets, when you're building real-time analytics pipelines, when you need to scale data processing across clusters—Apache Spark becomes your weapon of choice. Apache Spark is the foundation of modern big data processing. Without proper Spark understanding, you're building on shaky ground with inefficient processing, memory issues, and poor performance. This guide shows you how to wield Spark with the precision of a distributed systems engineer. @@ -961,4 +960,3 @@ Apache Spark mastery provides the foundation for distributed data processing. Wh --- -*This guide provides the complete machinery for mastering Apache Spark. The patterns scale from development to production, from simple transformations to complex machine learning pipelines.* diff --git a/docs/tutorials/database-data-engineering/duckdb-parquet-data-quality.md b/docs/tutorials/database-data-engineering/duckdb-parquet-data-quality.md index d016977..2c307cb 100644 --- a/docs/tutorials/database-data-engineering/duckdb-parquet-data-quality.md +++ b/docs/tutorials/database-data-engineering/duckdb-parquet-data-quality.md @@ -1,6 +1,5 @@ # DuckDB Parquet Data Quality Inspection: Finding Bad Data Files -**Objective**: Master DuckDB for comprehensive Parquet file inspection and data quality analysis. When you need to identify corrupted files, when you want to detect data anomalies, when you're building data quality pipelines—DuckDB becomes your weapon of choice. DuckDB provides the foundation for efficient Parquet file inspection and data quality analysis. Without proper understanding of file validation, data profiling, and anomaly detection, you're building unreliable data pipelines that miss the power of automated data quality assessment. This guide shows you how to wield DuckDB with the precision of a senior data engineer. @@ -1068,4 +1067,3 @@ DuckDB provides the foundation for efficient Parquet file inspection and data qu --- -*This guide provides the complete machinery for mastering DuckDB Parquet data quality inspection. The patterns scale from simple file validation to complex data quality pipelines, from basic integrity checks to advanced anomaly detection.* diff --git a/docs/tutorials/database-data-engineering/geospatial-knowledge-graph.md b/docs/tutorials/database-data-engineering/geospatial-knowledge-graph.md index 9d98b4d..d92f59d 100644 --- a/docs/tutorials/database-data-engineering/geospatial-knowledge-graph.md +++ b/docs/tutorials/database-data-engineering/geospatial-knowledge-graph.md @@ -1,8 +1,5 @@ # Geospatial Knowledge Graph: From Raw Geometry to Reasoning -**Objective**: Build a geospatial knowledge graph that understands place relationships. Not just points on a map, but connections: roads connecting cities, facilities within flood zones, sensor streams near pipelines, routes avoiding hazards. This tutorial combines PostGIS (heavy geometry math) with Neo4j (graph model + pathfinding/semantics) to create an intelligent geospatial system. - -When you need to understand spatial relationships at scale, when you want to query "shortest safe route avoiding flood polygons within 24h", when you're building location-aware applications that reason about connectivity and proximity—geospatial knowledge graphs become your weapon of choice. ## 0) Prerequisites (Read Once, Live by Them) @@ -1095,4 +1092,3 @@ Geospatial knowledge graphs provide the foundation for intelligent location-awar --- -*This guide provides the complete machinery for mastering geospatial knowledge graphs. The patterns scale from simple proximity analysis to complex spatial reasoning, from basic connectivity to advanced graph analytics.* diff --git a/docs/tutorials/database-data-engineering/go-osm-tiling-pipeline.md b/docs/tutorials/database-data-engineering/go-osm-tiling-pipeline.md index 56be937..40e5600 100644 --- a/docs/tutorials/database-data-engineering/go-osm-tiling-pipeline.md +++ b/docs/tutorials/database-data-engineering/go-osm-tiling-pipeline.md @@ -1,6 +1,5 @@ # Gluing the Real World: Go CLI for OSM → PostGIS → Tiles → Object Storage -**Objective**: A single Go binary conducts PBF fetch, checksum, PostGIS load, view materialization, tile generation/serving, and publishing. Concrete, durable, and debuggable. ## Architecture Diagram diff --git a/docs/tutorials/database-data-engineering/graph-vs-vector-databases.md b/docs/tutorials/database-data-engineering/graph-vs-vector-databases.md index 7f91baf..d61bbed 100644 --- a/docs/tutorials/database-data-engineering/graph-vs-vector-databases.md +++ b/docs/tutorials/database-data-engineering/graph-vs-vector-databases.md @@ -1,6 +1,5 @@ # Graph vs Vector Databases: When Relationships Meet Similarity -**Objective**: Master the fundamental differences between graph and vector databases for modern data applications. When you need to model explicit relationships, when you're building similarity search systems, when you need to understand the trade-offs between graph traversal and vector similarity—graph and vector databases become your weapons of choice. Graph and vector databases solve fundamentally different problems in data architecture. Without proper understanding of their strengths and limitations, you're building inefficient systems, choosing wrong technologies, and missing the power of hybrid approaches. This guide shows you how to wield both with the precision of a senior data engineer. @@ -1046,4 +1045,3 @@ Graph and vector databases solve fundamentally different problems in modern data --- -*This guide provides the complete machinery for mastering graph and vector databases. The patterns scale from simple queries to complex hybrid architectures, from development to production.* diff --git a/docs/tutorials/database-data-engineering/h3-raster-to-hex.md b/docs/tutorials/database-data-engineering/h3-raster-to-hex.md index cad8bf6..f3f6cbe 100644 --- a/docs/tutorials/database-data-engineering/h3-raster-to-hex.md +++ b/docs/tutorials/database-data-engineering/h3-raster-to-hex.md @@ -1,6 +1,5 @@ # H3 Raster to Hex: Converting Geospatial Rasters to Hexagonal Grids -**Objective**: Master the conversion of raster data (satellite imagery, elevation models, climate data) to H3 hexagonal grids for efficient spatial analysis, aggregation, and visualization. Transform pixels into hexagons for better spatial reasoning and analysis. ## Architecture @@ -847,4 +846,3 @@ docker compose --profile viz up -d --- -*This tutorial provides the complete machinery for converting raster data to H3 hexagonal grids. Each component is production-ready, copy-paste runnable, and designed to handle large-scale geospatial data processing with efficient spatial indexing and aggregation.* diff --git a/docs/tutorials/database-data-engineering/h3-tile38-nats-duckdb.md b/docs/tutorials/database-data-engineering/h3-tile38-nats-duckdb.md index 2fb0892..9250f05 100644 --- a/docs/tutorials/database-data-engineering/h3-tile38-nats-duckdb.md +++ b/docs/tutorials/database-data-engineering/h3-tile38-nats-duckdb.md @@ -1,6 +1,5 @@ # Hexes in Motion: H3 + Tile38 + NATS + DuckDB -**Objective**: Build a geo event bus where devices publish positions to NATS JetStream, Tile38 triggers geofence enter/exit events, DuckDB keeps a compact columnar history for offline analytics, and H3 powers hex aggregation for heatmaps. Real-time geofences with hex aggregation and offline analytics. ## Architecture @@ -624,4 +623,3 @@ docker compose exec tile38 bash /setup.sh --- -*This tutorial provides the complete machinery for real-time geofencing with hexagonal spatial analytics. Each component is production-ready, copy-paste runnable, and designed to handle high-frequency position updates with sub-millisecond geofence detection.* diff --git a/docs/tutorials/database-data-engineering/index.md b/docs/tutorials/database-data-engineering/index.md index ea5ac4b..3523d90 100644 --- a/docs/tutorials/database-data-engineering/index.md +++ b/docs/tutorials/database-data-engineering/index.md @@ -1,6 +1,5 @@ # Database & Data Engineering Tutorials -**Objective**: Master complex database and data engineering implementations through step-by-step guides. When you need to implement specific database technologies, when you want to follow proven patterns, when you need copy-paste runnable examples—these tutorials become your weapon of choice. ## PostgreSQL & PostGIS @@ -45,4 +44,3 @@ --- -*These tutorials provide the complete machinery for implementing key database and data engineering technologies and workflows. Each guide includes production-ready examples, configuration files, and best practices for enterprise deployment.* diff --git a/docs/tutorials/database-data-engineering/ipfs-surreal-meili-nats-deno-svelte.md b/docs/tutorials/database-data-engineering/ipfs-surreal-meili-nats-deno-svelte.md index 35c6f1f..f64dee2 100644 --- a/docs/tutorials/database-data-engineering/ipfs-surreal-meili-nats-deno-svelte.md +++ b/docs/tutorials/database-data-engineering/ipfs-surreal-meili-nats-deno-svelte.md @@ -1,6 +1,5 @@ # Content-Addressed Knowledge: IPFS + SurrealDB + Meilisearch Driven by NATS + Deno -**Objective**: Build a content-addressed knowledge system where documents arrive via HTTP (UI) → pushed onto NATS JetStream. A Deno worker consumes events, canonicalizes JSON, computes a CID (IPFS), pins bytes to Kubo (IPFS node), writes structured facts to SurrealDB, and indexes text to Meilisearch. The SvelteKit UI can search (Meili) → fetch canonical record (SurrealDB) → verify byte integrity via IPFS CID. ## Architecture @@ -826,4 +825,3 @@ docker compose --profile ops up -d --- -*This tutorial provides the complete machinery for content-addressed knowledge systems. Each component is production-ready, copy-paste runnable, and designed for verifiable, searchable, and immutable document storage with event-driven processing.* diff --git a/docs/tutorials/database-data-engineering/kafka-timescaledb-iot.md b/docs/tutorials/database-data-engineering/kafka-timescaledb-iot.md index 97c12e4..4c89c44 100644 --- a/docs/tutorials/database-data-engineering/kafka-timescaledb-iot.md +++ b/docs/tutorials/database-data-engineering/kafka-timescaledb-iot.md @@ -7,7 +7,6 @@ tags: # Streaming IoT Telemetry with Kafka + TimescaleDB (via Docker Compose) -**Objective**: Simulate thousands of device ticks (JSON/Avro), publish to Kafka, sink into Timescale hypertables using Kafka Connect JDBC Sink. Cover schemas, partitioning, retention, and performance. This tutorial builds a complete IoT data streaming pipeline that ingests device telemetry, processes it through Kafka, and stores it in TimescaleDB for time-series analytics. You'll learn production patterns for high-throughput data ingestion and real-time analytics. diff --git a/docs/tutorials/database-data-engineering/openmaptiles-us-dark-z12.md b/docs/tutorials/database-data-engineering/openmaptiles-us-dark-z12.md index 2502988..6942b77 100644 --- a/docs/tutorials/database-data-engineering/openmaptiles-us-dark-z12.md +++ b/docs/tutorials/database-data-engineering/openmaptiles-us-dark-z12.md @@ -1,6 +1,5 @@ # Generating Dark OpenMapTiles for the Entire US at Zoom Level 12 -**Objective**: Master generating dark-themed vector tiles for the United States using OpenMapTiles. When you need custom basemaps for dashboards, offline mapping, or air-gapped deployments—this tutorial provides a complete, reproducible pipeline. ## Introduction diff --git a/docs/tutorials/database-data-engineering/postgis-geometry-indexing.md b/docs/tutorials/database-data-engineering/postgis-geometry-indexing.md index 5b77a25..c2f2d24 100644 --- a/docs/tutorials/database-data-engineering/postgis-geometry-indexing.md +++ b/docs/tutorials/database-data-engineering/postgis-geometry-indexing.md @@ -7,7 +7,7 @@ tags: # PostGIS Geometry Indexing — Best Practices -This tutorial establishes the definitive approach to PostGIS geometry indexing that makes spatial queries fast, predictable, and boring. We enforce correct SRIDs, SARGable predicates, proper index classes (GiST/SP-GiST), ruthless query hygiene, and maintenance that doesn't lie. +This tutorial is how I set up PostGIS geometry indexes so spatial queries stay fast and predictable. We enforce correct SRIDs, SARGable predicates, proper index classes (GiST/SP-GiST), ruthless query hygiene, and maintenance that doesn't lie. **TL;DR:** GiST on geometry + clean predicates wins most of the time. Don't wrap the indexed column in functions. Use KNN for nearest neighbors. Partition only when you must. Verify with EXPLAIN (ANALYZE, BUFFERS) or you're telling yourself bedtime stories. diff --git a/docs/tutorials/database-data-engineering/postgres-pgaudit-pgcron-auditing.md b/docs/tutorials/database-data-engineering/postgres-pgaudit-pgcron-auditing.md index 4ab0613..e1c245e 100644 --- a/docs/tutorials/database-data-engineering/postgres-pgaudit-pgcron-auditing.md +++ b/docs/tutorials/database-data-engineering/postgres-pgaudit-pgcron-auditing.md @@ -1,6 +1,5 @@ # Auditing PostgreSQL with PgAudit and PgCron: Log Rotation, Top Users/Tables, and Activity Log Views -**Objective**: Master production-grade PostgreSQL auditing with PgAudit and PgCron. When you need comprehensive audit trails, automated log rotation, and analytical views of database activity—this tutorial provides a complete, reproducible setup. ## Introduction diff --git a/docs/tutorials/database-data-engineering/postgres-pooling.md b/docs/tutorials/database-data-engineering/postgres-pooling.md index 2529fb4..bce7a66 100644 --- a/docs/tutorials/database-data-engineering/postgres-pooling.md +++ b/docs/tutorials/database-data-engineering/postgres-pooling.md @@ -1,6 +1,5 @@ # Best Practices in PostgreSQL Connection Pooling with PgBouncer and FastAPI -**Objective**: Master PostgreSQL connection pooling to handle thousands of concurrent requests without overwhelming your database. When your FastAPI app scales beyond a few hundred users, when PostgreSQL connections become a bottleneck, when you need to optimize database resource usage—connection pooling becomes your weapon of choice. PostgreSQL connection pooling is the bridge between your application's concurrency model and the database's process-based architecture. Without proper pooling, you're flying blind into production with applications that could exhaust database connections and crash under load. This guide shows you how to wield both PgBouncer and application-level pooling with the precision of a seasoned database engineer. @@ -911,4 +910,3 @@ PostgreSQL connection pooling is the foundation of scalable database application --- -*This tutorial provides the complete machinery for mastering PostgreSQL connection pooling. The patterns scale from development to production, from simple applications to enterprise-grade deployments.* diff --git a/docs/tutorials/database-data-engineering/pulsar-flink-pinot-superset.md b/docs/tutorials/database-data-engineering/pulsar-flink-pinot-superset.md index c2e9c88..6383917 100644 --- a/docs/tutorials/database-data-engineering/pulsar-flink-pinot-superset.md +++ b/docs/tutorials/database-data-engineering/pulsar-flink-pinot-superset.md @@ -1,6 +1,5 @@ # Realtime Analytics: Pulsar → Flink → Pinot with Superset (2025 Edition) -**Objective**: Build a sub-second analytics stack: Pulsar streams data, Flink shapes it, Pinot serves OLAP queries, and Superset paints the truth. One compose.yml, toggle systems with profiles, and watch time bend. ## Architecture @@ -420,4 +419,3 @@ docker compose --profile viz up -d --- -*This tutorial provides the complete machinery for sub-second streaming analytics. Each component is production-ready, copy-paste runnable, and designed to handle real-world data volumes with millisecond query latencies.* diff --git a/docs/tutorials/database-data-engineering/real-time-data-processing.md b/docs/tutorials/database-data-engineering/real-time-data-processing.md index a94afd3..fe2f164 100644 --- a/docs/tutorials/database-data-engineering/real-time-data-processing.md +++ b/docs/tutorials/database-data-engineering/real-time-data-processing.md @@ -1,6 +1,5 @@ # Real-Time Data Processing with Kafka and TimescaleDB -**Objective**: Build a complete real-time data processing pipeline using Kafka for streaming and TimescaleDB for time-series storage. Real-time data processing enables immediate insights and responsive applications. This tutorial covers building a production-ready streaming pipeline that handles geospatial time-series data at scale. diff --git a/docs/tutorials/database-data-engineering/solr-postgres-jsonb-search.md b/docs/tutorials/database-data-engineering/solr-postgres-jsonb-search.md index 5f86640..61e3b61 100644 --- a/docs/tutorials/database-data-engineering/solr-postgres-jsonb-search.md +++ b/docs/tutorials/database-data-engineering/solr-postgres-jsonb-search.md @@ -1,6 +1,5 @@ # Solr + Postgres JSONB: Faceted, Full-Text, and Geo Search at Speed -**Objective**: Master the art of hybrid search—keeping relational truth in Postgres while unleashing Solr's full-text, faceting, and geospatial powers on JSONB data. When you need lightning-fast search with facets, suggestions, and geo queries, when you want to combine the reliability of Postgres with the search prowess of Solr—this tutorial becomes your weapon of choice. ## Overview @@ -810,4 +809,3 @@ curl "http://localhost:8983/solr/appcore/select?q=*:*&facet=true&facet.field=org --- -*This tutorial provides the complete machinery for hybrid Postgres-Solr search. Each pattern includes implementation examples, search strategies, and real-world usage patterns for enterprise search systems.* diff --git a/docs/tutorials/development-tools/find-files-parquet-fdw.md b/docs/tutorials/development-tools/find-files-parquet-fdw.md index 5d1c12e..4061f48 100644 --- a/docs/tutorials/development-tools/find-files-parquet-fdw.md +++ b/docs/tutorials/development-tools/find-files-parquet-fdw.md @@ -1,6 +1,5 @@ # A Customizable find_files Function for parquet_s3_fdw -**Objective**: Build a Python helper that discovers Parquet files across local storage, MinIO, Vast, and AWS S3, then generates clean SQL CREATE FOREIGN TABLE statements for parquet_s3_fdw. The pain of manually writing FDW table DDLs is real. This tutorial provides a machine that does the heavy lifting—discovering Parquet files and generating the SQL to expose them as foreign tables in Postgres. @@ -674,4 +673,3 @@ This `find_files` function eliminates the manual pain of creating foreign tables --- -*This tutorial provides the complete machinery for automating parquet_s3_fdw table creation. The machine handles the complexity, you get clean SQL.* diff --git a/docs/tutorials/development-tools/go-tech-mixer.md b/docs/tutorials/development-tools/go-tech-mixer.md index 704566f..239d53c 100644 --- a/docs/tutorials/development-tools/go-tech-mixer.md +++ b/docs/tutorials/development-tools/go-tech-mixer.md @@ -1,6 +1,5 @@ # Mixing PostGIS, Kafka, and Grafana with Go Glue -**Objective**: A strange but powerful tutorial: wire a geospatial DB, a streaming engine, and a monitoring UI together — controlled by a Go CLI. ## Why This Stack? diff --git a/docs/tutorials/development-tools/index.md b/docs/tutorials/development-tools/index.md index 1233a76..c4858b7 100644 --- a/docs/tutorials/development-tools/index.md +++ b/docs/tutorials/development-tools/index.md @@ -1,6 +1,5 @@ # Development Tools Tutorials -**Objective**: Master complex development tools implementations through step-by-step guides. When you need to implement specific development tools, when you want to follow proven patterns, when you need copy-paste runnable examples—these tutorials become your weapon of choice. ## JSON Processing @@ -26,4 +25,3 @@ --- -*These tutorials provide the complete machinery for implementing key development tools and workflows. Each guide includes production-ready examples, configuration files, and best practices for enterprise deployment.* diff --git a/docs/tutorials/development-tools/jq-json-parsing-mastery.md b/docs/tutorials/development-tools/jq-json-parsing-mastery.md index 391cce9..98b5702 100644 --- a/docs/tutorials/development-tools/jq-json-parsing-mastery.md +++ b/docs/tutorials/development-tools/jq-json-parsing-mastery.md @@ -1,6 +1,5 @@ # The Art of JSON Parsing with jq: A Gonzo Journey Through Data Manipulation -**Objective**: Master the dark art of JSON parsing with `jq`—from Docker Bake outputs to API responses, from simple filtering to complex transformations that would make a data scientist weep with joy. Welcome to the wild frontier of JSON manipulation, where `jq` reigns supreme as the Swiss Army knife of data parsing. This isn't just a tutorial—it's a gonzo expedition into the heart of structured data, where we'll emerge victorious over the most complex JSON nightmares you can throw at us. diff --git a/docs/tutorials/development-tools/mosquitto-mqtt-python.md b/docs/tutorials/development-tools/mosquitto-mqtt-python.md index 328b938..7f717f3 100644 --- a/docs/tutorials/development-tools/mosquitto-mqtt-python.md +++ b/docs/tutorials/development-tools/mosquitto-mqtt-python.md @@ -1,6 +1,5 @@ # Mosquitto MQTT Best Practices & Python Clients -**Objective**: Master MQTT messaging with Mosquitto broker and Python clients for reliable, production-ready IoT and event-driven applications. When you need to connect sensors, devices, and services, when you need real-time messaging with guaranteed delivery, when you need to scale to thousands of connections—MQTT becomes your weapon of choice. MQTT (Message Queuing Telemetry Transport) is the bridge between your IoT devices and your applications. Without proper MQTT setup, you're flying blind into production with messaging that could lose data, overwhelm your broker, or fail under load. This guide shows you how to wield Mosquitto and Python MQTT clients with the precision of a seasoned DevOps engineer. @@ -1246,4 +1245,3 @@ MQTT messaging with Mosquitto and Python provides reliable, scalable communicati --- -*This tutorial provides the complete machinery for mastering MQTT messaging with Mosquitto and Python. The patterns scale from development to production, from simple sensors to enterprise-grade IoT systems.* diff --git a/docs/tutorials/development-tools/python-modbus-devices.md b/docs/tutorials/development-tools/python-modbus-devices.md index e365f17..9f2618e 100644 --- a/docs/tutorials/development-tools/python-modbus-devices.md +++ b/docs/tutorials/development-tools/python-modbus-devices.md @@ -1,6 +1,5 @@ # Industrial Data Harvesting: Python Modbus Device Communication -**Objective**: Master industrial device communication using Modbus protocol to connect to PLCs, sensors, and automation equipment. When you need to integrate with industrial systems, when you want to monitor manufacturing equipment, when you're building IoT solutions for industrial environments—Python Modbus becomes your weapon of choice. Industrial devices speak a different language. Let's learn to communicate with PLCs, HMIs, and sensors using the universal language of industrial automation: Modbus protocol. @@ -1194,4 +1193,3 @@ Python Modbus requires understanding both industrial protocols and Python implem --- -*This tutorial provides the complete machinery for Python Modbus device communication. The patterns scale from simple device reading to complex industrial automation, from basic protocol implementation to advanced device management.* diff --git a/docs/tutorials/development-tools/python-udp.md b/docs/tutorials/development-tools/python-udp.md index 18c9ae8..d509162 100644 --- a/docs/tutorials/development-tools/python-udp.md +++ b/docs/tutorials/development-tools/python-udp.md @@ -1,6 +1,5 @@ # Reading & Writing UDP Messages with Python -**Objective**: Master UDP (User Datagram Protocol) messaging with Python for high-performance, low-latency applications. When you need fast, connectionless communication, when you're building real-time systems, when you need to handle thousands of concurrent connections—UDP becomes your weapon of choice. UDP is the foundation of fast, connectionless networking. Without proper UDP understanding, you're building on shaky ground with unreliable message delivery, buffer overflows, and performance bottlenecks. This guide shows you how to wield Python's socket and asyncio libraries with the precision of a network engineer. @@ -1090,4 +1089,3 @@ UDP messaging with Python provides fast, connectionless communication for real-t --- -*This tutorial provides the complete machinery for mastering UDP messaging with Python. The patterns scale from development to production, from simple clients to enterprise-grade real-time systems.* diff --git a/docs/tutorials/development-tools/tauri-rqlite-syncthing.md b/docs/tutorials/development-tools/tauri-rqlite-syncthing.md index 44f094e..b8df314 100644 --- a/docs/tutorials/development-tools/tauri-rqlite-syncthing.md +++ b/docs/tutorials/development-tools/tauri-rqlite-syncthing.md @@ -1,6 +1,5 @@ # Hermit Sync: Local-first Apps with rqlite + Syncthing -**Objective**: Build a cross-platform desktop app (Tauri) that uses rqlite (Raft-replicated SQLite over HTTP) for local durability while Syncthing P2P syncs encrypted snapshots between machines. Result: tiny, resilient, air-gap-friendly CRUD with zero central server. ## Architecture @@ -653,4 +652,3 @@ npm run tauri dev --- -*This tutorial provides the complete machinery for offline-first desktop apps with peer-to-peer sync. Each component is production-ready, copy-paste runnable, and designed for air-gap-friendly deployment without central infrastructure.* diff --git a/docs/tutorials/diagrams/layered-systems-diagrams-mermaid-to-svg.md b/docs/tutorials/diagrams/layered-systems-diagrams-mermaid-to-svg.md index a046524..40140a0 100644 --- a/docs/tutorials/diagrams/layered-systems-diagrams-mermaid-to-svg.md +++ b/docs/tutorials/diagrams/layered-systems-diagrams-mermaid-to-svg.md @@ -9,7 +9,6 @@ tags: # Layered Systems Diagrams: Mermaid Source → SVG Artifact (End-to-End Example) -**Objective**: Build a small layered diagram set (Context + Workflow) for an IoT → MQTT → Data Lake → Dashboard system, keep Mermaid as source, render to SVG, and embed both in MkDocs. --- @@ -18,7 +17,7 @@ tags: By the end of this tutorial you will: 1. Have two diagram sources (context + workflow) under `docs/assets/diagrams/examples/`. -2. Know how to render them to SVG using the site’s existing tooling in `tools/diagrams/`. +2. Know how to render them to SVG with `@mermaid-js/mermaid-cli` (via the scripts in `tools/diagrams/` in this repo). 3. Embed both Mermaid (source) and SVG (artifact) in a doc so readers see source and output. 4. Apply the layering and boundary discipline from [Systems Diagramming Best Practices](../../best-practices/diagrams/systems-diagramming-best-practices.md). @@ -126,7 +125,7 @@ Again, the `.mmd` file is only the Mermaid code (no ` ```mermaid ` wrapper). ## 5. Render to SVG -The repo already provides Mermaid → SVG tooling in `tools/diagrams/`. No new tooling is required. +Render with the Node scripts in `tools/diagrams/` (wrapping `mmdc`), or any equivalent `mmdc` invocation pointed at your `.mmd` files. 1. **Install dependencies** (once): @@ -149,7 +148,7 @@ The repo already provides Mermaid → SVG tooling in `tools/diagrams/`. No new t node render_mermaid_svg.mjs --input ../../docs/assets/diagrams/examples ``` -After a successful run, commit both the `.mmd` and `.svg` files so the site can serve the SVG without a build step. +After a successful run, commit both the `.mmd` and `.svg` files if your documentation host serves static assets without running Mermaid at build time. --- @@ -209,6 +208,5 @@ The SVG will scale cleanly; the Mermaid block will render in the browser via MkD ## 8. See also - **[Systems Diagramming Best Practices](../../best-practices/diagrams/systems-diagramming-best-practices.md)** — Layering, boundaries, entropy control, and pattern library. -- **[Generating Complex Workflow Diagrams as SVG](../../best-practices/diagrams/svg-workflow-generation.md)** — Repository conventions and style for Mermaid → SVG. -- **[Mermaid → SVG Workflow Pipeline](mermaid-to-svg-workflow-pipeline.md)** — Full pipeline and `data-platform-workflow` example. -- **[Diagram Style Guide](../../diagrams/style-guide.md)** — Mermaid vs SVG, orientation, and accessibility. +- **[Generating Complex Workflow Diagrams as SVG](../../best-practices/diagrams/svg-workflow-generation.md)** — Layout conventions and artifact workflow. +- **[Rendering Mermaid diagrams to SVG](mermaid-to-svg-workflow-pipeline.md)** — Pipeline setup and `data-platform-workflow` example. diff --git a/docs/tutorials/diagrams/mermaid-to-svg-workflow-pipeline.md b/docs/tutorials/diagrams/mermaid-to-svg-workflow-pipeline.md index 5aef1b0..d716226 100644 --- a/docs/tutorials/diagrams/mermaid-to-svg-workflow-pipeline.md +++ b/docs/tutorials/diagrams/mermaid-to-svg-workflow-pipeline.md @@ -7,7 +7,7 @@ tags: - tooling --- -# Mermaid → SVG Workflow Pipeline (From Diagram Source to Published Artifact) +# Rendering Mermaid diagrams to SVG --- @@ -19,7 +19,7 @@ This tutorial walks you through the full pipeline for producing a committed, ren 2. A Node.js render script that produces a `.svg` artifact next to each source 3. A committed `.svg` that renders in GitHub Pages and MkDocs Material without a build step -This is the workflow used throughout this site. Once set up, adding a new diagram takes three steps: write the `.mmd`, run the render script, commit both files. +Once set up, adding a new diagram is usually: write the `.mmd`, run the render script, commit both source and SVG. --- @@ -75,7 +75,7 @@ node --version ### 2. Write or Edit the `.mmd` Source -Create or edit a file under `docs/assets/diagrams/`. Use the `%%{init}%%` directive to set rendering options, and follow the style guide for orientation, subgraph boundaries, and node text length. +Create or edit a file under your diagrams directory. Use the `%%{init}%%` directive for rendering options; prefer `flowchart LR` for pipelines and short node labels (see [SVG workflow best practices](../../best-practices/diagrams/svg-workflow-generation.md)). Example structure for a new workflow diagram: @@ -285,6 +285,6 @@ flowchart LR --- !!! tip "See also" - - [SVG Workflow Generation Best Practices](../../best-practices/diagrams/svg-workflow-generation.md) — principles, conventions, and the full style guide for Mermaid-first diagram authoring - - [Diagram Style Guide](../../diagrams/style-guide.md) — reusable Mermaid snippet library and formatting rules - - [ADR 0015: Standardize Diagrams on Mermaid](../../adr/0015-diagrams-mermaid.md) — the architectural decision behind this pipeline + - [SVG Workflow Generation Best Practices](../../best-practices/diagrams/svg-workflow-generation.md) — principles and layout conventions for Mermaid-first diagrams + - [Systems Diagramming Best Practices](../../best-practices/diagrams/systems-diagramming-best-practices.md) — layering, boundaries, and pattern snippets + - [Layered Systems Diagrams: Mermaid → SVG](layered-systems-diagrams-mermaid-to-svg.md) — worked example with context and workflow diagrams diff --git a/docs/tutorials/docker-infrastructure/ansible-dask-heterogeneous.md b/docs/tutorials/docker-infrastructure/ansible-dask-heterogeneous.md index cefa646..4e74471 100644 --- a/docs/tutorials/docker-infrastructure/ansible-dask-heterogeneous.md +++ b/docs/tutorials/docker-infrastructure/ansible-dask-heterogeneous.md @@ -1,6 +1,5 @@ # Provisioning a Dask Cluster with Ansible (Scheduler + CPU/GPU Workers) -**Objective**: Master Dask cluster deployment for heterogeneous compute workloads. When you need to process large datasets with mixed CPU/GPU tasks, when you want to leverage RAPIDS for GPU-accelerated analytics, when you're building production data processing infrastructure—Dask cluster deployment becomes your weapon of choice. Dask enables distributed computing across CPU and GPU workers with intelligent task scheduling. This tutorial shows you how to wield Dask cluster deployment with the precision of a battle-hardened cluster engineer, covering scheduler setup, heterogeneous worker pools, and production operations. @@ -1106,4 +1105,3 @@ Dask cluster deployment requires understanding both distributed computing patter --- -*This tutorial provides the complete machinery for Dask cluster deployment with Ansible. The patterns scale from simple CPU-only clusters to complex heterogeneous GPU-accelerated deployments, from basic task scheduling to advanced resource management and monitoring.* diff --git a/docs/tutorials/docker-infrastructure/ansible-rke2-rancher-pgo-prefect.md b/docs/tutorials/docker-infrastructure/ansible-rke2-rancher-pgo-prefect.md index 6731c47..cb1082e 100644 --- a/docs/tutorials/docker-infrastructure/ansible-rke2-rancher-pgo-prefect.md +++ b/docs/tutorials/docker-infrastructure/ansible-rke2-rancher-pgo-prefect.md @@ -1,6 +1,5 @@ # Building a 2×8 RKE2 Cluster with Rancher, PGO, and Prefect Using Ansible -**Objective**: Master automated Kubernetes cluster provisioning with Ansible. When you need to provision production-grade RKE2 clusters, when you want Rancher for cluster management, when you need Postgres operators and workflow orchestration—this tutorial becomes your weapon of choice. When you're tired of hand-taming clusters and want a machine to build other machines, this is the path: Ansible rides herd on bare nodes, RKE2 welds them into a cluster, Rancher gives you a GUI throne, PGO breeds Postgres instances like bioengineered livestock, and Prefect runs your flows in the middle of it all. @@ -912,5 +911,4 @@ watch -n 5 'KUBECONFIG=~/.kube/config-rke2 kubectl get nodes,pods -A' --- -*This tutorial provides the complete machinery for automated Kubernetes cluster provisioning. The patterns scale from development to production, from simple clusters to enterprise-grade deployments with full observability and workflow orchestration.* diff --git a/docs/tutorials/docker-infrastructure/ansible-slurm-raspberrypi.md b/docs/tutorials/docker-infrastructure/ansible-slurm-raspberrypi.md index 3e7fe0a..4486928 100644 --- a/docs/tutorials/docker-infrastructure/ansible-slurm-raspberrypi.md +++ b/docs/tutorials/docker-infrastructure/ansible-slurm-raspberrypi.md @@ -1,6 +1,5 @@ # Deploying SLURM on a Raspberry Pi Cluster with Ansible -**Objective**: Master cluster automation with Ansible and SLURM on Raspberry Pi hardware. When you need to learn HPC concepts, when you want reproducible cluster deployments, when you're building educational or development clusters—Ansible becomes your weapon of choice. SLURM (Simple Linux Utility for Resource Management) is the de facto standard for job scheduling in HPC environments. Raspberry Pi clusters provide an affordable way to learn HPC concepts, and Ansible makes cluster deployment reproducible and scalable. This tutorial shows you how to wield Ansible with the precision of a cluster administrator. @@ -995,4 +994,3 @@ SLURM cluster deployment requires understanding both HPC concepts and infrastruc --- -*This tutorial provides the complete machinery for deploying SLURM clusters with Ansible. The patterns scale from simple educational clusters to complex production HPC environments, from basic job scheduling to advanced resource management.* diff --git a/docs/tutorials/docker-infrastructure/compose-profiles-polyglot-stack.md b/docs/tutorials/docker-infrastructure/compose-profiles-polyglot-stack.md index 8d8e30b..196bf8b 100644 --- a/docs/tutorials/docker-infrastructure/compose-profiles-polyglot-stack.md +++ b/docs/tutorials/docker-infrastructure/compose-profiles-polyglot-stack.md @@ -1,6 +1,5 @@ # Compose Profiles at Scale: Toggling a Polyglot Data/Geo/ML City -**Objective**: One compose.yml. Many realities. Flip `--profile gpu` and torch tensors; `--profile tiles` and mint vector tiles. Profiles as power, not garnish. ## Architecture & Profiles Map diff --git a/docs/tutorials/docker-infrastructure/harbor-registry-setup.md b/docs/tutorials/docker-infrastructure/harbor-registry-setup.md index ebe7f8f..a658629 100644 --- a/docs/tutorials/docker-infrastructure/harbor-registry-setup.md +++ b/docs/tutorials/docker-infrastructure/harbor-registry-setup.md @@ -1,6 +1,5 @@ # Harbor Container Registry Setup: Kubernetes & Docker Compose with Network Storage -**Objective**: Master Harbor container registry deployment with production-grade storage and security. When you need enterprise container registry capabilities, when you want vulnerability scanning and image signing, when you're building secure CI/CD pipelines—Harbor becomes your weapon of choice. Harbor is an open-source container registry that provides enterprise-grade features including vulnerability scanning, image signing, RBAC, and replication. This tutorial shows you how to deploy Harbor with the precision of a DevOps engineer, covering both Kubernetes and Docker Compose environments with network-attached storage. @@ -1301,4 +1300,3 @@ Harbor deployment requires understanding both container orchestration and storag --- -*This tutorial provides the complete machinery for deploying Harbor with network storage. The patterns scale from simple single-node deployments to complex enterprise clusters, from basic container storage to advanced security and compliance.* diff --git a/docs/tutorials/docker-infrastructure/index.md b/docs/tutorials/docker-infrastructure/index.md index 0a64b6d..7a86936 100644 --- a/docs/tutorials/docker-infrastructure/index.md +++ b/docs/tutorials/docker-infrastructure/index.md @@ -1,6 +1,5 @@ # Docker & Infrastructure Tutorials -**Objective**: Master complex infrastructure implementations through step-by-step guides. When you need to implement specific infrastructure technologies, when you want to follow proven patterns, when you need copy-paste runnable examples—these tutorials become your weapon of choice. ## Containerization & Orchestration @@ -25,4 +24,3 @@ --- -*These tutorials provide the complete machinery for implementing key infrastructure technologies and workflows. Each guide includes production-ready examples, configuration files, and best practices for enterprise deployment.* diff --git a/docs/tutorials/docker-infrastructure/multistage-conda-to-scratch.md b/docs/tutorials/docker-infrastructure/multistage-conda-to-scratch.md index 7f3d32c..e09ec5c 100644 --- a/docs/tutorials/docker-infrastructure/multistage-conda-to-scratch.md +++ b/docs/tutorials/docker-infrastructure/multistage-conda-to-scratch.md @@ -7,7 +7,6 @@ tags: # Multi-Stage Docker: Conda Build → Slim Runtime → scratch -**Objective**: Keep Conda's dependency-resolution ergonomics for building, then ship a final image that starts from `scratch` — zero OS layer, zero shell, zero package manager. You copy in exactly what your app needs to run and nothing else. scratch is where convenience goes to die. You must supply every file your process will ever open: the Python interpreter, the Conda environment tree, every shared library it links against, TLS certificates if you hit HTTPS, and a minimal `/etc/passwd` if your app checks user identity. Miss one and you get a cryptic runtime crash. Get it right and your image is 80–95% smaller than the Conda base and has almost no CVE surface. diff --git a/docs/tutorials/docker-infrastructure/rke2-raspberry-pi.md b/docs/tutorials/docker-infrastructure/rke2-raspberry-pi.md index 28b1632..770ce55 100644 --- a/docs/tutorials/docker-infrastructure/rke2-raspberry-pi.md +++ b/docs/tutorials/docker-infrastructure/rke2-raspberry-pi.md @@ -5,11 +5,11 @@ tags: - docker --- -# RKE2 on a Raspberry Pi Farm +# RKE2 on a Raspberry Pi lab cluster -**Objective**: Build a production-ready Kubernetes cluster using RKE2 on Raspberry Pi hardware. Transform your Pi collection into a mini data center with enterprise-grade orchestration. +I use RKE2 on ARM64 Pi hardware when I want a small Kubernetes cluster that behaves enough like the larger systems I work with to be useful for experiments. This is a **lab** setup — not a reference production deployment. -Why use RKE2? ARM64 support, lightweight, secure defaults, manageable at scale. Why Raspberry Pi? Affordable lab cluster to learn Kubernetes orchestration without breaking the bank. This is your gateway to understanding how real data centers work. +RKE2 is a reasonable fit here: ARM64 builds exist, the install is small, and you get a real kubelet/API without assembling the control plane by hand. Pis are cheap, loud, and thermally honest, which is fine for learning and integration tests. ## 1) Hardware Prep: The Foundation @@ -114,28 +114,24 @@ sudo hostnamectl set-hostname rpi-worker-2 sudo hostnamectl set-hostname rpi-worker-3 ``` -**Why These Tools**: Essential for debugging and monitoring. You'll need them when things go wrong. +### Static IP configuration -### Static IP Configuration +Kubernetes join tokens and firewall rules are easier when node addresses do not drift. On **Raspberry Pi OS Bookworm and later**, networking is usually **NetworkManager**, not `dhcpcd`. ```bash -# Edit /etc/dhcpcd.conf -sudo vim /etc/dhcpcd.conf +# List connections — names vary by image +nmcli -t -f NAME con show -# Add at the end: -interface eth0 -static ip_address=192.168.1.10/24 # Master -static ip_address=192.168.1.11/24 # Worker 1 -static ip_address=192.168.1.12/24 # Worker 2 -static ip_address=192.168.1.13/24 # Worker 3 -static routers=192.168.1.1 -static domain_name_servers=192.168.1.1 8.8.8.8 - -# Restart networking -sudo systemctl restart dhcpcd +# Example: static IPv4 on the wired profile (adjust NAME and address per node) +sudo nmcli con mod "Wired connection 1" \ + ipv4.method manual \ + ipv4.addresses 192.168.1.10/24 \ + ipv4.gateway 192.168.1.1 \ + ipv4.dns "192.168.1.1 8.8.8.8" +sudo nmcli con up "Wired connection 1" ``` -**Why Static IPs**: Kubernetes needs predictable networking. DHCP can cause cluster instability. +Legacy images still on `dhcpcd` can use `/etc/dhcpcd.conf` static blocks — verify with `systemctl status NetworkManager` before editing the wrong stack. ### Enable cgroups (Critical for Kubernetes) @@ -143,8 +139,8 @@ sudo systemctl restart dhcpcd # Edit /boot/firmware/cmdline.txt sudo vim /boot/firmware/cmdline.txt -# Add to the end of the line: -cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory +# Append to the single kernel cmdline (do not add a second line): +cgroup_enable=cpuset cgroup_enable=memory # Reboot sudo reboot @@ -691,7 +687,6 @@ RKE2 on Raspberry Pi creates a powerful, affordable Kubernetes learning environm --- -*This tutorial provides the complete machinery for building a production-ready RKE2 cluster on Raspberry Pi hardware. The cluster scales from learning to production, from megabytes to terabytes.* !!! tip "See also" - [Pi-Based Sample Library Server](../just-for-fun/pi-sample-server.md) — another production-quality Pi project; deploy the API on your RKE2 cluster diff --git a/docs/tutorials/docker-infrastructure/slim-geospatial-gpu-conda.md b/docs/tutorials/docker-infrastructure/slim-geospatial-gpu-conda.md index a67721f..5e6595e 100644 --- a/docs/tutorials/docker-infrastructure/slim-geospatial-gpu-conda.md +++ b/docs/tutorials/docker-infrastructure/slim-geospatial-gpu-conda.md @@ -7,7 +7,6 @@ tags: # Slim Geospatial + GPU Containers: Conda (GDAL) → Production Image Without 12 GB Bloat -**Objective**: Build a Python geospatial + optional GPU stack with Conda (including GDAL, PROJ, GEOS), then ship a production image under 2–4 GB by being deliberate about every layer. No 12 GB monsters. No mystery bloat. No hand-waving about "just use conda-pack." --- diff --git a/docs/tutorials/docker-infrastructure/slim-gpu-docker-images.md b/docs/tutorials/docker-infrastructure/slim-gpu-docker-images.md index 1d6cf47..9cfff6c 100644 --- a/docs/tutorials/docker-infrastructure/slim-gpu-docker-images.md +++ b/docs/tutorials/docker-infrastructure/slim-gpu-docker-images.md @@ -1,6 +1,5 @@ # Slimming Down GPU-Based Docker Images -**Objective**: Transform bloated GPU containers from quarter-ton dev packages to fighting weight production images. Cut cold start times, reduce storage costs, and maintain CUDA compatibility. When your container is dragging a quarter-ton of dev packages and orphaned toolchains, you're paying for cold starts with your soul. This guide shows how to cut GPU images down to fighting weight without breaking CUDA. @@ -483,4 +482,3 @@ GPU containers are expensive to build and run. Every MB saved is money saved and --- -*This tutorial provides the complete machinery for building slim GPU containers. The patterns scale from megabytes to terabytes, from development to production.* diff --git a/docs/tutorials/docker-infrastructure/slim-tf-gpu-images.md b/docs/tutorials/docker-infrastructure/slim-tf-gpu-images.md index 624a52e..5c69e05 100644 --- a/docs/tutorials/docker-infrastructure/slim-tf-gpu-images.md +++ b/docs/tutorials/docker-infrastructure/slim-tf-gpu-images.md @@ -1,6 +1,5 @@ # Slimming Down GPU-Enabled TensorFlow Images -**Objective**: Transform bloated TensorFlow GPU containers from gigabytes to megabytes. Cut cold start times, reduce storage costs, and maintain CUDA compatibility while keeping your models running fast. Big images bleed time and money. Cut your TensorFlow GPU containers to size without breaking CUDA. This guide shows you how to build lean, mean inference machines that start fast and run efficiently. @@ -511,4 +510,3 @@ TensorFlow GPU containers are bloated by default. The key is understanding what --- -*This tutorial provides the complete machinery for building slim TensorFlow GPU containers. The patterns scale from development to production, from megabytes to terabytes.* diff --git a/docs/tutorials/docker-infrastructure/zfs-tank-nvme.md b/docs/tutorials/docker-infrastructure/zfs-tank-nvme.md index 832320c..920c0dd 100644 --- a/docs/tutorials/docker-infrastructure/zfs-tank-nvme.md +++ b/docs/tutorials/docker-infrastructure/zfs-tank-nvme.md @@ -1,6 +1,5 @@ # Building a ZFS Tank with OS on NVMe -**Objective**: Build a production-ready ZFS storage system with OS on fast NVMe and data on a resilient ZFS pool. Transform your storage from fragile to bulletproof. Why NVMe for OS? Speed + separation of concerns. Why ZFS tank? Snapshots, checksums, compression, resilience. This is how you build storage that doesn't die when drives fail. @@ -691,4 +690,3 @@ ZFS transforms your storage from fragile to bulletproof. With OS on NVMe and dat --- -*This tutorial provides the complete machinery for building a production-ready ZFS storage system. The tank scales from terabytes to petabytes, from home labs to data centers.* diff --git a/docs/tutorials/embedded/esp32-eink-sensor-monitor.md b/docs/tutorials/embedded/esp32-eink-sensor-monitor.md index 8935be2..39f50a5 100644 --- a/docs/tutorials/embedded/esp32-eink-sensor-monitor.md +++ b/docs/tutorials/embedded/esp32-eink-sensor-monitor.md @@ -6,7 +6,6 @@ tags: # ESP32 E-Ink Environmental Monitor (Low Power + Safe Design) -**Objective**: Build a battery-powered environmental sensor node that reads temperature, humidity, pressure, and light every 5 minutes, renders a clean dashboard on a 1.54" e-ink display, then deep-sleeps. Target: >7 days on a 2000 mAh LiPo. --- diff --git a/docs/tutorials/embedded/esp32-mqtt-home-assistant-integration.md b/docs/tutorials/embedded/esp32-mqtt-home-assistant-integration.md index 02fbb7e..1bb4991 100644 --- a/docs/tutorials/embedded/esp32-mqtt-home-assistant-integration.md +++ b/docs/tutorials/embedded/esp32-mqtt-home-assistant-integration.md @@ -7,7 +7,6 @@ tags: # ESP32 + MQTT + Home Assistant Integration (Secure by Design) -**Objective**: Connect an ESP32 sensor node to Home Assistant via a TLS-secured Mosquitto MQTT broker. The ESP32 publishes BME280 temperature/humidity readings and responds to LED toggle commands. HA auto-discovers the device. Everything is authenticated and encrypted. --- diff --git a/docs/tutorials/embedded/esp32-rf-room-light-controller.md b/docs/tutorials/embedded/esp32-rf-room-light-controller.md index a469e5a..1d5e296 100644 --- a/docs/tutorials/embedded/esp32-rf-room-light-controller.md +++ b/docs/tutorials/embedded/esp32-rf-room-light-controller.md @@ -6,7 +6,6 @@ tags: # ESP32 RF Room Light Controller (LCD + Rotary Encoder + Button + 433 MHz RF) -**Objective**: Build a wall-mounted ESP32 controller that selects a room with a rotary encoder, toggles lights over 433 MHz RF, and displays current state on a 16×2 I2C LCD — all from a clean, non-blocking firmware. No cloud. No app. A physical knob on a wall that works instantly. diff --git a/docs/tutorials/go-development/index.md b/docs/tutorials/go-development/index.md index 71d912a..8b9d9a4 100644 --- a/docs/tutorials/go-development/index.md +++ b/docs/tutorials/go-development/index.md @@ -1,6 +1,5 @@ # Go Development Tutorials -**Objective**: Master complex Go implementations through step-by-step guides. When you need to implement specific Go technologies, when you want to follow proven patterns, when you need copy-paste runnable examples—these tutorials become your weapon of choice. ## Core Go @@ -8,4 +7,3 @@ --- -*These tutorials provide the complete machinery for implementing key Go technologies and workflows. Each guide includes production-ready examples, configuration files, and best practices for enterprise deployment.* diff --git a/docs/tutorials/index.md b/docs/tutorials/index.md index febdcf9..3f8359c 100644 --- a/docs/tutorials/index.md +++ b/docs/tutorials/index.md @@ -2,91 +2,29 @@ Step-by-step builds: prerequisites, commands, config files, and something you can reproduce on your own machine. -If you already know *what* you want to run and mainly need *how*, start here. For principles and trade-offs, see [Best Practices](../best-practices/index.md). +For principles and trade-offs, see [Best Practices](../best-practices/index.md). -## 🚀 Quick Start +## Quick start -Getting started guides for essential technologies and workflows. +- [Quick start overview](quick-start/index.md) -- **[Quick Start Overview](quick-start/index.md)** - Getting started with essential technologies +## By topic -## 🐍 Python Development +- [Python](python-development/index.md) +- [Rust](rust-development/index.md) +- [Go](go-development/index.md) +- [Docker and infrastructure](docker-infrastructure/index.md) — includes [RKE2 on Raspberry Pi](docker-infrastructure/rke2-raspberry-pi.md) +- [Database and data engineering](database-data-engineering/index.md) — includes [PostGIS geometry indexing](database-data-engineering/postgis-geometry-indexing.md) +- [Machine learning and AI](ml-ai/index.md) +- [System administration](system-administration/index.md) +- [Data science and visualization](data-science-visualization/index.md) +- [Development tools](development-tools/index.md) +- [Diagrams](diagrams/layered-systems-diagrams-mermaid-to-svg.md) -Python development tutorials covering core Python, web development, and advanced patterns. +## Just for Fun -- **[Python Development Overview](python-development/index.md)** - Core Python, web development, and advanced patterns +Creative and experimental builds — [overview](just-for-fun/index.md). -## 🦀 Rust Development +## Archived integration walkthroughs -Rust development tutorials for systems programming and high-performance applications. - -- **[Rust Development Overview](rust-development/index.md)** - Systems programming and high-performance applications - -## 🐳 Docker & Infrastructure - -Infrastructure tutorials covering containerization, orchestration, and automation. - -- **[Docker & Infrastructure Overview](docker-infrastructure/index.md)** - Containerization, orchestration, and automation - -## 🗄️ Database & Data Engineering - -Database and data engineering tutorials for PostgreSQL, analytics, and big data processing. - -- **[Database & Data Engineering Overview](database-data-engineering/index.md)** - PostgreSQL, analytics, and big data processing - -## 🤖 Machine Learning & AI - -Machine learning and AI tutorials for ML operations, model deployment, and AI integration. - -- **[Machine Learning & AI Overview](ml-ai/index.md)** - ML operations, model deployment, and AI integration - -## 🔧 System Administration - -System administration tutorials for network management, workflow orchestration, and Unix tools. - -- **[System Administration Overview](system-administration/index.md)** - Network management, orchestration, and Unix tools - -## 📊 Data Science & Visualization - -Data science and visualization tutorials for Jupyter notebooks, diagrams, and generative art. - -- **[Data Science & Visualization Overview](data-science-visualization/index.md)** - Jupyter notebooks, diagrams, and generative art - -## 🛠️ Development Tools - -Development tools tutorials for JSON processing, file management, and messaging systems. - -- **[Development Tools Overview](development-tools/index.md)** - JSON processing, file management, and messaging - -## 🖼️ Diagrams - -Layered systems diagrams: Mermaid source to SVG artifact and workflow pipeline. - -- **[Layered Systems Diagrams: Mermaid → SVG](diagrams/layered-systems-diagrams-mermaid-to-svg.md)** — End-to-end example (Context + Workflow) with IoT → MQTT → Lakehouse -- **[Mermaid → SVG Workflow Pipeline](diagrams/mermaid-to-svg-workflow-pipeline.md)** — Full pipeline and render tooling - -## 🎨 Just for Fun - -Creative and experimental tutorials for creative solutions and experimental implementations. - -- **[Just for Fun Overview](just-for-fun/index.md)** - Creative solutions and experimental implementations - -## 🔗 Best Practices Integration - -Comprehensive tutorials that combine multiple best practices into complete, production-ready systems. - -- **[Best Practices Integration Overview](best-practices-integration/index.md)** - Integrated solutions combining multiple best practices -- **[Event-Driven Microservices with Observability and Data Governance](best-practices-integration/event-driven-microservices-observability-data-governance.md)** - Event-driven architecture with observability and data governance -- **[Multi-Environment Deployment with Secrets and Configuration Governance](best-practices-integration/multi-environment-deployment-secrets-config-governance.md)** - Secure multi-environment deployments with configuration management -- **[Data Pipeline with Quality Governance and Observability](best-practices-integration/data-pipeline-quality-governance-observability.md)** - Data pipelines with comprehensive quality governance -- **[Resilient Microservices with Chaos Engineering and Operational Resilience](best-practices-integration/resilient-microservices-chaos-engineering.md)** - Resilient systems with chaos engineering validation -- **[Secure Polyglot System with Identity Federation and Encryption Lifecycle](best-practices-integration/secure-polyglot-identity-encryption.md)** - Secure systems with identity federation and encryption management -- **[Cost-Optimized ML Pipeline with Capacity Planning and Data Retention](best-practices-integration/cost-optimized-ml-pipeline-capacity-retention.md)** - Cost-efficient ML pipelines with capacity planning -- **[Multi-Region Disaster Recovery with Temporal Governance and Blast Radius Reduction](best-practices-integration/multi-region-dr-temporal-blast-radius.md)** - Multi-region DR with time synchronization -- **[Developer Experience Platform with System Taxonomy and Repository Standardization](best-practices-integration/developer-experience-taxonomy-repository.md)** - DX platform with standardization -- **[Progressive Delivery System with Release Management and Fitness Functions](best-practices-integration/progressive-delivery-release-fitness.md)** - Safe deployments with quality gates -- **[Geospatial Data Mesh with Cost Optimization and Capacity Planning](best-practices-integration/geospatial-data-mesh-cost-capacity.md)** - Geospatial data mesh with cost optimization - ---- - -*When a tutorial assumes background you do not have, back up to the linked best-practice page or the section overview.* \ No newline at end of file +The former **Best Practices Integration** series (multi-topic “production system” composites) lived under `tutorials/best-practices-integration/`. See [Archived: Best Practices Integration](best-practices-integration/index.md) for status. Prefer focused tutorials above and judgment pages in Best Practices. diff --git a/docs/tutorials/ml-ai/index.md b/docs/tutorials/ml-ai/index.md index 6dae335..1b36bb9 100644 --- a/docs/tutorials/ml-ai/index.md +++ b/docs/tutorials/ml-ai/index.md @@ -1,6 +1,5 @@ # Machine Learning & AI Tutorials -**Objective**: Master complex machine learning and AI implementations through step-by-step guides. When you need to implement specific ML/AI technologies, when you want to follow proven patterns, when you need copy-paste runnable examples—these tutorials become your weapon of choice. ## ML Operations @@ -16,4 +15,3 @@ --- -*These tutorials provide the complete machinery for implementing key machine learning and AI technologies and workflows. Each guide includes production-ready examples, configuration files, and best practices for enterprise deployment.* diff --git a/docs/tutorials/ml-ai/local-llm-deployments.md b/docs/tutorials/ml-ai/local-llm-deployments.md index 0eeaa87..e9c64f2 100644 --- a/docs/tutorials/ml-ai/local-llm-deployments.md +++ b/docs/tutorials/ml-ai/local-llm-deployments.md @@ -1,6 +1,5 @@ # Local LLM Deployments: Methods, Quantization, and OpenAI-Compatible Serving -**Objective**: A ruthless field guide to running modern LLMs on your own hardware. CPU to GPU, tiny to obnoxious, with real benchmarks and a path to RAG. ## Mental Model: What You're Actually Choosing @@ -545,4 +544,3 @@ docker compose --profile bench up --build --- -*This tutorial provides the complete machinery for running LLMs locally. Each component is production-ready, copy-paste runnable, and designed for real-world deployment with proper trade-offs and failure handling.* diff --git a/docs/tutorials/ml-ai/mcp-mlflow-toolchain.md b/docs/tutorials/ml-ai/mcp-mlflow-toolchain.md index a7c741c..08fa23b 100644 --- a/docs/tutorials/ml-ai/mcp-mlflow-toolchain.md +++ b/docs/tutorials/ml-ai/mcp-mlflow-toolchain.md @@ -1,6 +1,5 @@ # MCP ↔ MLflow Toolchain: Data-Agnostic LLM Experimentation -**Objective**: Build a complete MCP toolchain that enables LLMs to run MLflow experiments on any data source—files, SQL databases, or inline context. When you need to give LLMs safe access to machine learning workflows, when you want to track experiments across different data sources, when you're building AI-driven data science platforms—MCP becomes your weapon of choice. MCP provides the foundation for secure, controlled access to MLflow experiments. Without proper understanding of tool design, data source abstraction, and experiment tracking, you're building vulnerable systems that miss the power of controlled ML experimentation. This guide shows you how to wield MCP with the precision of a senior data engineer. @@ -1192,4 +1191,3 @@ MCP-MLflow provides the foundation for secure, scalable AI-driven machine learni --- -*This guide provides the complete machinery for mastering MCP-MLflow toolchains. The patterns scale from simple experiment tracking to complex AI-driven ML workflows, from basic security to advanced production deployment.* diff --git a/docs/tutorials/ml-ai/mlflow-api-experiments.md b/docs/tutorials/ml-ai/mlflow-api-experiments.md index 50cfef2..8af40d1 100644 --- a/docs/tutorials/ml-ai/mlflow-api-experiments.md +++ b/docs/tutorials/ml-ai/mlflow-api-experiments.md @@ -1,6 +1,5 @@ # MLflow API Experiments: Models, Inference, and Database Integration -**Objective**: Master MLflow's REST API to run experiments, manage models, and output inference to databases. Build production-ready ML pipelines that track experiments, serve models, and persist predictions. When your ML models need to scale beyond notebooks and into production, MLflow's APIs become your bridge between experimentation and deployment. This guide shows you how to leverage MLflow's REST API for experiment tracking, model management, and inference serving with database integration. @@ -1288,4 +1287,3 @@ MLflow APIs provide the foundation for production ML workflows. When configured --- -*This tutorial provides the complete machinery for building production-ready ML workflows with MLflow APIs. The patterns scale from development to production, from single experiments to enterprise deployments.* diff --git a/docs/tutorials/ml-ai/onnx-browser-inference.md b/docs/tutorials/ml-ai/onnx-browser-inference.md index 7f8af58..8805fb2 100644 --- a/docs/tutorials/ml-ai/onnx-browser-inference.md +++ b/docs/tutorials/ml-ai/onnx-browser-inference.md @@ -6,7 +6,6 @@ tags: # ONNX Browser Inference: Deploy Machine Learning Models Directly in the Browser -**Objective**: Master ONNX model inference in web browsers for client-side machine learning applications. When you need to run ML models directly in the browser, when you're building privacy-preserving applications, when you want to reduce server costs and latency—browser-based ONNX inference becomes your weapon of choice. Browser-based ONNX inference enables client-side machine learning without server dependencies. Without proper understanding of browser limitations and optimization techniques, you're building slow, memory-intensive applications that fail on mobile devices. This guide shows you how to deploy ONNX models in browsers with the precision of a web ML engineer. @@ -1031,7 +1030,6 @@ Browser-based ONNX inference enables client-side machine learning without server --- -*This guide provides the complete machinery for mastering browser ONNX inference. The patterns scale from simple demos to production web applications, from basic inference to advanced optimization.* !!! tip "See also" - [ML Systems Architecture Governance](../../best-practices/ml-ai/ml-systems-architecture-governance.md) — production model serving patterns; where ONNX fits in a full ML platform diff --git a/docs/tutorials/ml-ai/rag-ollama-db.md b/docs/tutorials/ml-ai/rag-ollama-db.md index acc69c8..046f8d5 100644 --- a/docs/tutorials/ml-ai/rag-ollama-db.md +++ b/docs/tutorials/ml-ai/rag-ollama-db.md @@ -1,6 +1,5 @@ # Retrieval-Augmented Generation on Databases with Ollama -**Objective**: Build a RAG pipeline that bridges your structured database with Ollama's LLM, enabling natural language queries over your data. When you need to query databases in natural language, when you want to augment LLMs with live data, when you're building intelligent data interfaces—RAG on databases becomes your weapon of choice. RAG on databases is the foundation of intelligent data interaction. Without proper understanding of retrieval patterns, embedding strategies, and prompt engineering, you're building static interfaces that miss the power of conversational data access. This guide shows you how to wield RAG with the precision of a senior applied ML engineer. @@ -1325,4 +1324,3 @@ RAG on databases provides the foundation for intelligent data interaction. When --- -*This guide provides the complete machinery for mastering RAG on databases with Ollama. The patterns scale from simple queries to complex reasoning, from basic retrieval to advanced generation.* diff --git a/docs/tutorials/ml-ai/semantic-ml-training.md b/docs/tutorials/ml-ai/semantic-ml-training.md index 3ae0e10..b11b9b7 100644 --- a/docs/tutorials/ml-ai/semantic-ml-training.md +++ b/docs/tutorials/ml-ai/semantic-ml-training.md @@ -1,6 +1,5 @@ # Training Semantic Machine Learning Models: From Embeddings to Production -**Objective**: Master the complete pipeline for training semantic machine learning models that understand meaning, context, and relationships in text data. When you need to build intelligent search systems, when you want to create recommendation engines, when you're developing AI applications that understand language—semantic ML becomes your weapon of choice. Semantic machine learning provides the foundation for understanding meaning in text data. Without proper understanding of embeddings, vector spaces, and semantic relationships, you're building systems that miss the power of contextual understanding. This guide shows you how to wield semantic ML with the precision of a senior ML engineer. @@ -1209,4 +1208,3 @@ Semantic machine learning provides the foundation for understanding meaning in t --- -*This guide provides the complete machinery for mastering semantic machine learning. The patterns scale from simple word embeddings to complex contextual understanding, from basic similarity to advanced semantic reasoning.* diff --git a/docs/tutorials/python-development/advanced-nicegui-architecture.md b/docs/tutorials/python-development/advanced-nicegui-architecture.md index a5dadbb..6224f79 100644 --- a/docs/tutorials/python-development/advanced-nicegui-architecture.md +++ b/docs/tutorials/python-development/advanced-nicegui-architecture.md @@ -1,39 +1,19 @@ -# Advanced NiceGUI Architecture: Persistent Backends, Components, Spinners, and Real-Time UI +# Advanced NiceGUI: backends, components, and live updates -**Objective**: Master production-grade NiceGUI applications with persistent backends, reusable components, real-time updates, and advanced async patterns. When you need to build scalable, maintainable UI applications that integrate with Redis, PostgreSQL, and external APIs—this tutorial becomes your weapon of choice. +NiceGUI examples often stop at a single file with buttons on a page. When I keep a UI around, I usually need **shared services** (database pools, Redis clients), **components** I can reuse, and **clear startup/shutdown** so connections do not leak across reloads. -## Introduction +This tutorial walks through **one layout** that has worked for me. It is not official NiceGUI doctrine — verify lifecycle hooks and APIs against the [NiceGUI docs](https://nicegui.io/) for your version. -Most NiceGUI tutorials stop at "here's how to make a button." Real applications need persistent database connections, connection pooling, real-time updates, loading states, error handling, and reusable components. This tutorial covers the architecture patterns that transform NiceGUI from a prototyping tool into a production-capable framework. +## What this covers -**What "Advanced NiceGUI" Means**: +- Class-based pages and small UI components +- Long-lived connection pools (PostgreSQL, Redis) tied to app startup +- Loading states and streaming output for slow work +- Pub/sub or polling patterns for live UI updates -- **Persistent Backends**: Connection pools for Redis and PostgreSQL that survive across page navigations -- **Reusable Components**: Class-based UI components that encapsulate behavior and styling -- **Real-Time Updates**: Redis pub/sub and WebSocket patterns for live UI updates -- **Loading States**: Spinners, progress bars, and streaming output for long-running operations -- **Lifecycle Management**: Startup/shutdown handlers, graceful connection cleanup, background tasks -- **State Management**: Session state, global state, and reactive variables that sync across components +## Project structure -**Why Class-Based Pages Matter**: - -Single-file NiceGUI apps work for demos. Production apps need: -- Separation of concerns (pages, components, services) -- Dependency injection (shared services, connection pools) -- Testability (mockable services, isolated components) -- Maintainability (clear structure, reusable patterns) - -**How Backend Pools Improve Performance**: - -Creating new connections for every request is slow and wasteful. Connection pools: -- Reuse existing connections (10-100x faster) -- Limit concurrent connections (prevent resource exhaustion) -- Handle reconnection automatically (resilience) -- Provide connection health monitoring (observability) - -## Project Structure - -Here's a production-ready directory layout that scales: +Example layout (adjust to your app): ``` advanced-nicegui-app/ @@ -1536,5 +1516,4 @@ This architecture provides a solid foundation for building production-grade Nice --- -*This tutorial provides the complete machinery for building production-grade NiceGUI applications. The patterns scale from prototypes to enterprise systems, from simple pages to complex real-time dashboards with multiple backend integrations.* diff --git a/docs/tutorials/python-development/chaos-engineering-k8s-python.md b/docs/tutorials/python-development/chaos-engineering-k8s-python.md index d8ccb5e..9a8a561 100644 --- a/docs/tutorials/python-development/chaos-engineering-k8s-python.md +++ b/docs/tutorials/python-development/chaos-engineering-k8s-python.md @@ -1,6 +1,5 @@ # Chaos Engineering with Kubernetes and Python -**Objective**: Master chaos engineering to build resilient distributed systems through controlled failure injection. When you need to verify system resilience, when you want to prevent production disasters, when you're building fault-tolerant applications—chaos engineering becomes your weapon of choice. Chaos engineering is not "breaking things for fun"—it's controlled fault injection to verify resilience. This tutorial shows you how to wield chaos engineering with the precision of a seasoned reliability engineer, covering everything from basic pod kills to advanced network chaos and observability integration. @@ -1110,4 +1109,3 @@ Chaos engineering requires understanding both system resilience and controlled f --- -*This tutorial provides the complete machinery for chaos engineering. The patterns scale from simple pod kills to complex distributed system chaos, from basic resilience testing to advanced failure scenario analysis.* diff --git a/docs/tutorials/python-development/click-to-fastapi-conversion.md b/docs/tutorials/python-development/click-to-fastapi-conversion.md index cd64be3..6afa6df 100644 --- a/docs/tutorials/python-development/click-to-fastapi-conversion.md +++ b/docs/tutorials/python-development/click-to-fastapi-conversion.md @@ -1,6 +1,5 @@ # Click CLI to FastAPI Conversion: The Machine's Reckoning -**Objective**: Automatically convert Click-based CLI hierarchies into fully functional FastAPI applications, preserving command semantics as HTTP endpoints while maintaining the dark art of parameter mapping. This isn't just a conversion—it's a systematic dismantling of CLI paradigms and their reconstruction as HTTP interfaces. The machine will do the heavy lifting, but you'll understand every cut it makes. @@ -929,4 +928,3 @@ This isn't just a conversion tool—it's a systematic approach to bridging the g --- -*This tutorial provides the complete machinery for converting Click CLI hierarchies into FastAPI endpoints. The machine handles the complexity, but you control the transformation.* diff --git a/docs/tutorials/python-development/distributed-nicegui-redis.md b/docs/tutorials/python-development/distributed-nicegui-redis.md index 5c480b4..7ec8806 100644 --- a/docs/tutorials/python-development/distributed-nicegui-redis.md +++ b/docs/tutorials/python-development/distributed-nicegui-redis.md @@ -1,6 +1,5 @@ # Distributed NiceGUI Architecture with Redis: Horizontal Scaling and Real-Time Updates -**Objective**: Master building horizontally scalable NiceGUI applications using Redis as the central nervous system. When you need to run multiple NiceGUI instances across machines, coordinate real-time updates, and scale workers independently—this tutorial becomes your weapon of choice. ## Introduction @@ -1619,5 +1618,4 @@ async def _subscribe_with_retry(self): --- -*This tutorial provides the complete machinery for building horizontally scalable NiceGUI applications. The patterns scale from single instances to distributed clusters, from prototypes to enterprise systems with real-time coordination across multiple machines.* diff --git a/docs/tutorials/python-development/index.md b/docs/tutorials/python-development/index.md index aeee74a..8b361c8 100644 --- a/docs/tutorials/python-development/index.md +++ b/docs/tutorials/python-development/index.md @@ -1,6 +1,5 @@ # Python Development Tutorials -**Objective**: Master complex Python implementations through step-by-step guides. When you need to implement specific Python technologies, when you want to follow proven patterns, when you need copy-paste runnable examples—these tutorials become your weapon of choice. ## Core Python @@ -20,4 +19,3 @@ --- -*These tutorials provide the complete machinery for implementing key Python technologies and workflows. Each guide includes production-ready examples, configuration files, and best practices for enterprise deployment.* diff --git a/docs/tutorials/python-development/nicegui-class-based-pages.md b/docs/tutorials/python-development/nicegui-class-based-pages.md index f575e46..17d6c7b 100644 --- a/docs/tutorials/python-development/nicegui-class-based-pages.md +++ b/docs/tutorials/python-development/nicegui-class-based-pages.md @@ -1,6 +1,5 @@ # Class-Based NiceGUI Pages and Integrations: A Practical Guide -**Objective**: Master building structured NiceGUI applications with class-based pages, dependency injection, and clean architecture. When you need to build real applications—not prototypes—with auth, dashboards, API integrations, and maintainable code, this tutorial becomes your weapon of choice. Most NiceGUI examples are single-file fever dreams: everything in `main.py`, global state drifting around like cosmic dust. That's fine for a quick prototype, but the moment you want a real app—auth, dashboards, API integrations—you need structure. @@ -1124,5 +1123,4 @@ class Config: --- -*This tutorial provides the complete machinery for building structured NiceGUI applications. The patterns scale from prototypes to production, from simple pages to complex dashboards with multiple backend integrations.* diff --git a/docs/tutorials/python-development/psycopg2-to-psycopg3-migration.md b/docs/tutorials/python-development/psycopg2-to-psycopg3-migration.md index d6792b2..6799897 100644 --- a/docs/tutorials/python-development/psycopg2-to-psycopg3-migration.md +++ b/docs/tutorials/python-development/psycopg2-to-psycopg3-migration.md @@ -1,6 +1,5 @@ # Migrating from psycopg2 to psycopg ≥ 3: A Brutally Practical Guide -**Objective**: Migrate your psycopg2 codebase to psycopg 3 with surgical precision. When your codebase is glued together with psycopg2, when you need async, safer adapters, better COPY, and cleaner ergonomics—psycopg 3 is the modern rewrite. This guide walks you through the migration with minimal downtime and no mystery meat. You've got a codebase glued together with psycopg2. It works—until you need async, safer adapters, better COPY, and cleaner ergonomics. psycopg 3 is the modern rewrite. This tutorial walks you—surgically—through the migration with minimal downtime and no mystery meat. @@ -793,4 +792,3 @@ psycopg 3 is the modern rewrite of psycopg2 with better performance, native asyn --- -*This tutorial provides the complete machinery for migrating from psycopg2 to psycopg 3. The patterns scale from simple scripts to production web applications.* diff --git a/docs/tutorials/python-development/r-shiny-geoapp.md b/docs/tutorials/python-development/r-shiny-geoapp.md index b01b8cb..d9a2d0e 100644 --- a/docs/tutorials/python-development/r-shiny-geoapp.md +++ b/docs/tutorials/python-development/r-shiny-geoapp.md @@ -1,6 +1,5 @@ # Building & Deploying a Geospatial R Shiny App (for Free) -**Objective**: Build an interactive world map where clicking countries toggles them red (visited), then deploy it online for free. R Shiny transforms R into a web application framework. We'll create "Where Have You Been in the World?"—a clickable world map that tracks visited countries, then deploy it to the cloud without spending a dime. diff --git a/docs/tutorials/python-development/ruff-check-ignore-pyproject.md b/docs/tutorials/python-development/ruff-check-ignore-pyproject.md index cd13cfd..fbfc527 100644 --- a/docs/tutorials/python-development/ruff-check-ignore-pyproject.md +++ b/docs/tutorials/python-development/ruff-check-ignore-pyproject.md @@ -1,6 +1,5 @@ # Ignoring Ruff Check Errors in pyproject.toml -**Objective**: Master the art of selectively ignoring Ruff check errors in pyproject.toml. Handle specific files, error types, and maintain code quality while working with Python projects. When your Python project includes generated code, third-party bindings, or legacy code, you need fine-grained control over which Ruff check errors to ignore. This guide shows you how to configure pyproject.toml to ignore specific errors, files, and patterns while maintaining code quality. @@ -812,4 +811,3 @@ Ruff check error ignoring in pyproject.toml provides fine-grained control over c --- -*This tutorial provides the complete machinery for configuring Ruff check error ignoring in pyproject.toml. The patterns scale from development to production, from simple scripts to complex multi-package projects.* diff --git a/docs/tutorials/python-development/websocket-chat-fastapi.md b/docs/tutorials/python-development/websocket-chat-fastapi.md index 6c03ec6..aefdfdb 100644 --- a/docs/tutorials/python-development/websocket-chat-fastapi.md +++ b/docs/tutorials/python-development/websocket-chat-fastapi.md @@ -1,6 +1,5 @@ # Building a Real-time Chat Application with WebSockets and FastAPI -**Objective**: Build a production-ready real-time chat application using FastAPI, WebSockets, and modern web technologies. When you need real-time communication, when you want to understand WebSocket patterns, when you're building interactive applications—WebSocket chat becomes your weapon of choice. Real-time chat applications require understanding both WebSocket mechanics and modern web development patterns. This tutorial shows you how to build a complete chat system with the precision of a senior full-stack engineer, covering everything from basic WebSocket handling to production deployment. @@ -1082,4 +1081,3 @@ WebSocket chat applications require understanding both real-time communication a --- -*This tutorial provides the complete machinery for WebSocket chat applications. The patterns scale from simple messaging to complex real-time systems, from basic WebSocket handling to advanced production deployment.* diff --git a/docs/tutorials/quick-start/creating-mkdocs-github-site.md b/docs/tutorials/quick-start/creating-mkdocs-github-site.md index 9550170..61697f5 100644 --- a/docs/tutorials/quick-start/creating-mkdocs-github-site.md +++ b/docs/tutorials/quick-start/creating-mkdocs-github-site.md @@ -88,7 +88,7 @@ nav: - Docker & Compose: best-practices/docker-and-compose.md - Tutorials: - Creating MkDocs GitHub Site: tutorials/creating-mkdocs-github-site.md - - Contact & Collaboration: getting-started.md + - Contact: getting-started.md ``` **Why:** Hierarchical navigation enables organized content structure. Nested sections group related content while maintaining clear information architecture. diff --git a/docs/tutorials/quick-start/index.md b/docs/tutorials/quick-start/index.md index 90656cd..390b746 100644 --- a/docs/tutorials/quick-start/index.md +++ b/docs/tutorials/quick-start/index.md @@ -1,6 +1,5 @@ # Quick Start Tutorials -**Objective**: Get up and running quickly with essential technologies and workflows. When you need to implement specific technologies fast, when you want to follow proven patterns, when you need copy-paste runnable examples—these tutorials become your weapon of choice. ## Getting Started @@ -9,4 +8,3 @@ --- -*These tutorials provide the complete machinery for getting started with key technologies and workflows. Each guide includes complete code examples, configuration files, and production-ready patterns.* diff --git a/docs/tutorials/rust-development/index.md b/docs/tutorials/rust-development/index.md index dfe170a..74fc61e 100644 --- a/docs/tutorials/rust-development/index.md +++ b/docs/tutorials/rust-development/index.md @@ -1,6 +1,5 @@ # Rust Development Tutorials -**Objective**: Master complex Rust implementations through step-by-step guides. When you need to implement specific Rust technologies, when you want to follow proven patterns, when you need copy-paste runnable examples—these tutorials become your weapon of choice. ## Systems Programming @@ -10,4 +9,3 @@ --- -*These tutorials provide the complete machinery for implementing key Rust technologies and workflows. Each guide includes production-ready examples, configuration files, and best practices for enterprise deployment.* diff --git a/docs/tutorials/rust-development/rust-csr-parquet-db.md b/docs/tutorials/rust-development/rust-csr-parquet-db.md index e95f548..8ebffc0 100644 --- a/docs/tutorials/rust-development/rust-csr-parquet-db.md +++ b/docs/tutorials/rust-development/rust-csr-parquet-db.md @@ -1,8 +1,5 @@ # Rust + CSR: Parse, Build from Parquet/DB, and Bend Sparse Data to Your Will -**Objective**: Master CSR (Compressed Sparse Row) matrices in Rust. Parse them, build them from Parquet files and databases, understand why they matter, and wield them for sparse linear algebra that would make dense matrices weep. - -When your sparse data exceeds memory by orders of magnitude, when your graph algorithms need cache-friendly iteration, when your recommenders demand O(nnz) complexity—CSR becomes your weapon of choice. This guide shows you how to build, parse, and optimize CSR matrices in Rust with surgical precision. ## 0) Prerequisites (Read Once, Live by Them) @@ -976,4 +973,3 @@ CSR matrices are the foundation of sparse linear algebra. When configured proper --- -*This tutorial provides the complete machinery for building, parsing, and optimizing CSR matrices in Rust. The patterns scale from development to production, from small matrices to massive sparse datasets.* diff --git a/docs/tutorials/rust-development/rust-event-sourcing.md b/docs/tutorials/rust-development/rust-event-sourcing.md index ed414b2..3266ee6 100644 --- a/docs/tutorials/rust-development/rust-event-sourcing.md +++ b/docs/tutorials/rust-development/rust-event-sourcing.md @@ -1,6 +1,5 @@ # Event-Sourcing in Rust with Async Streams and RocksDB -**Objective**: Build a high-performance event-sourcing engine that treats the log as the source of truth. When you need audit trails, when you want to replay history, when you're building systems where state is derived from events—event sourcing becomes your weapon of choice. Event sourcing flips persistence—don't store the final state, store every event. State is a derived view. This tutorial shows you how to wield event sourcing with the precision of a provocative systems engineer, covering everything from async event ingestion to high-performance projections and benchmarking. @@ -1227,4 +1226,3 @@ Event sourcing requires understanding both domain modeling and system architectu --- -*This tutorial provides the complete machinery for event sourcing in Rust. The patterns scale from simple event storage to complex distributed systems, from basic projections to advanced streaming architectures.* diff --git a/docs/tutorials/system-administration/awk-unix-text-processing.md b/docs/tutorials/system-administration/awk-unix-text-processing.md index dba40cf..2a60a77 100644 --- a/docs/tutorials/system-administration/awk-unix-text-processing.md +++ b/docs/tutorials/system-administration/awk-unix-text-processing.md @@ -1,6 +1,5 @@ # AWK Unix Text Processing Mastery: Parse ls, ps aux, and System Data Like a Pro -**Objective**: Master AWK for parsing Unix command output, system data, and text processing. When you need to extract specific data from `ls`, `ps aux`, `df`, `netstat`, and other Unix utilities, when you're building system monitoring scripts, when you need to process log files and structured text—AWK becomes your weapon of choice. AWK is the Swiss Army knife of Unix text processing. Without proper AWK mastery, you're parsing text inefficiently, writing complex shell scripts, and missing the power of pattern-action programming. This guide shows you how to wield AWK with the precision of a Unix systems engineer. @@ -996,4 +995,3 @@ AWK mastery provides the foundation for Unix text processing. When used correctl --- -*This guide provides the complete machinery for mastering AWK text processing. The patterns scale from simple field extraction to complex system monitoring, from basic scripts to production-grade data processing.* diff --git a/docs/tutorials/system-administration/index.md b/docs/tutorials/system-administration/index.md index f0f0990..9ab2ae2 100644 --- a/docs/tutorials/system-administration/index.md +++ b/docs/tutorials/system-administration/index.md @@ -1,6 +1,5 @@ # System Administration Tutorials -**Objective**: Master complex system administration implementations through step-by-step guides. When you need to implement specific system administration technologies, when you want to follow proven patterns, when you need copy-paste runnable examples—these tutorials become your weapon of choice. ## Network & Boot Management @@ -17,4 +16,3 @@ --- -*These tutorials provide the complete machinery for implementing key system administration technologies and workflows. Each guide includes production-ready examples, configuration files, and best practices for enterprise deployment.* diff --git a/docs/tutorials/system-administration/ipxe-multi-boot.md b/docs/tutorials/system-administration/ipxe-multi-boot.md index 528d67f..53cc211 100644 --- a/docs/tutorials/system-administration/ipxe-multi-boot.md +++ b/docs/tutorials/system-administration/ipxe-multi-boot.md @@ -1,6 +1,5 @@ # Creating iPXE Boot Images for Multi-System Network Booting -**Objective**: Boot multiple OSes over the network using iPXE, no more juggling USB sticks. Set up a single server that can boot any Linux distribution, rescue utilities, or custom images across your entire infrastructure. When you're managing dozens of servers, workstations, or lab machines, USB sticks become a bottleneck. iPXE lets you boot any OS over the network with a single server. This guide shows you how to build a complete network booting infrastructure that scales from a single lab to enterprise deployment. @@ -821,4 +820,3 @@ iPXE network booting transforms how you manage systems. Instead of juggling USB --- -*This tutorial provides the complete machinery for building iPXE network booting infrastructure. The patterns scale from development to production, from single machines to enterprise deployments.* diff --git a/docs/tutorials/system-administration/prefect-fifo-redis.md b/docs/tutorials/system-administration/prefect-fifo-redis.md index c2cbc88..6b89eb3 100644 --- a/docs/tutorials/system-administration/prefect-fifo-redis.md +++ b/docs/tutorials/system-administration/prefect-fifo-redis.md @@ -1,6 +1,5 @@ # Greedy FIFO Prefect Flow with Redis-backed Parameters -**Objective**: Implement a FIFO greedy scheduler pattern in Prefect where a parent flow continuously pulls parameter sets from Redis, and child flows run concurrently in a greedy, FIFO manner. Transform your orchestration from sequential to parallel while maintaining order. Why FIFO? Ensures jobs are processed in order of arrival, keeps GPUs/CPUs hot by dispatching immediately to idle workers. This pattern scales from single machines to distributed clusters while maintaining job order and maximizing resource utilization. @@ -1024,4 +1023,3 @@ FIFO greedy scheduling with Prefect and Redis creates a robust, scalable orchest --- -*This tutorial provides the complete machinery for building production-ready FIFO greedy orchestration systems. The patterns scale from single machines to distributed clusters.* diff --git a/docs/tutorials/system-administration/remote-dev-tmux-screen.md b/docs/tutorials/system-administration/remote-dev-tmux-screen.md index b1b6556..83ecf28 100644 --- a/docs/tutorials/system-administration/remote-dev-tmux-screen.md +++ b/docs/tutorials/system-administration/remote-dev-tmux-screen.md @@ -1,6 +1,5 @@ # Remote Development with tmux & screen: Best Practices for When the Wire Goes Dead -**Objective**: Master terminal multiplexers for bulletproof remote development that survives network chaos, SSH drops, and your own judgment failures. When you need to keep long-running processes alive, when you want to maintain context across network interruptions, when you're working on critical deployments—tmux and screen become your weapons of choice. Terminal multiplexers provide the foundation for resilient remote development. Without proper understanding of session management, persistence patterns, and network failure recovery, you're building fragile systems that miss the power of continuous productivity. This guide shows you how to wield tmux and screen with the precision of a senior systems engineer. @@ -974,4 +973,3 @@ Terminal multiplexers provide the foundation for resilient remote development. W --- -*This guide provides the complete machinery for mastering remote development with tmux and screen. The patterns scale from simple session management to complex collaborative workflows, from basic persistence to advanced automation.* diff --git a/docs/whats-new.md b/docs/whats-new.md index b6c1453..dfaa8d2 100644 --- a/docs/whats-new.md +++ b/docs/whats-new.md @@ -13,66 +13,51 @@ Edited when I add something worth mentioning — not a git log. ## February 2026 -### Diagrams & Tooling +### Diagrams & tooling -- **[SVG Workflow Generation Best Practices](best-practices/diagrams/svg-workflow-generation.md)** — Mermaid-first, artifact-driven approach to producing committed SVG diagrams: diagram-as-code principles, repo conventions, style guide, CI-ready rendering workflow, and a full data platform example. +- **[SVG Workflow Generation Best Practices](best-practices/diagrams/svg-workflow-generation.md)** — Mermaid-first, artifact-driven approach to committed SVG diagrams: diagram-as-code principles, rendering workflow, CI checks, and a data platform example. -- **[Mermaid → SVG Workflow Pipeline (Tutorial)](tutorials/diagrams/mermaid-to-svg-workflow-pipeline.md)** — Step-by-step guide to setting up `@mermaid-js/mermaid-cli`, rendering `.mmd` sources to `.svg` artifacts, and embedding them in MkDocs pages. Includes troubleshooting for font, viewBox, and Puppeteer issues. +- **[Mermaid → SVG Workflow Pipeline (Tutorial)](tutorials/diagrams/mermaid-to-svg-workflow-pipeline.md)** — Set up `@mermaid-js/mermaid-cli`, render `.mmd` sources to `.svg`, and embed figures in static documentation. Includes troubleshooting for font, viewBox, and Puppeteer issues. -- **[Diagram Style Guide](diagrams/style-guide.md)** — When to use Mermaid vs SVG, diagram type selection by content, formatting conventions (orientation, subgraphs, node text limits), accessibility requirements, and a reusable Mermaid snippet library (control plane/data plane, pipeline stages, monolith vs microservices, MQTT broker, ML training vs serving). +- **[Layered Systems Diagrams: Mermaid → SVG](tutorials/diagrams/layered-systems-diagrams-mermaid-to-svg.md)** — Context and workflow diagrams for an IoT → MQTT → lakehouse example, from source through rendered SVG. -- **[ADR 0015: Standardize Diagrams on Mermaid](adr/0015-diagrams-mermaid.md)** — Mermaid becomes the site standard for architecture, flow, state, and sequence diagrams; SVG is reserved for geospatial illustrations and cases where Mermaid's layout engine is insufficient. +### Reader guides -### Site Doctrine & Structure +- **[Architectural Compass](start-here-architectural-paths.md)** — Problem-oriented paths for embedded systems, data pipelines, microservices, ML, geospatial, and infrastructure. -- **[Start Here — Architectural Compass](start-here-architectural-paths.md)** — Decision-tree navigation organized by problem domain: embedded systems, data pipelines, microservices, ML systems, geospatial, and infrastructure. Each path sequences 4–6 essays with context. +- **[Reading Tracks](reading-tracks.md)** — Curated sequences: Modern Data Architecture, Distributed Systems & Scale, Embedded Systems, Observability & Operations, Infrastructure Economics, and Spatial Systems. -- **[Reading Tracks](reading-tracks.md)** — Six curated reading sequences: Modern Data Architecture, Distributed Systems & Scale, Embedded Systems Doctrine, Observability & Operations, Infrastructure Economics, and Spatial Systems. +- **[Decision Frameworks](decision-frameworks.md)** — Summaries from nine deep dives: Kubernetes, Microservices, Serverless, Real-Time, Storage, Analytical Systems, Metadata Governance, ML Deployment, GPU Infrastructure, and Spatial Architecture. -- **[Decision Frameworks](decision-frameworks.md)** — Aggregated decision frameworks from nine deep dives, each summarized in 5–10 lines: Kubernetes, Microservices, Serverless, Real-Time, Storage, Analytical Systems, Metadata Governance, ML Deployment, GPU Infrastructure, and Spatial Architecture. +- **[Anti-Patterns Index](anti-patterns.md)** — Premature Microservices, Overusing Kubernetes, Real-Time by Default, Data Swamp Formation, Serverless Cargo Cult, and Distributed Systems for Small Teams. -- **[Anti-Patterns Index](anti-patterns.md)** — Six architectural anti-patterns with diagnostic signals and deep dive links: Premature Microservices, Overusing Kubernetes, Real-Time by Default, Data Swamp Formation, Serverless Cargo Cult, and Distributed Systems for Small Teams. +- **[Systems engineering principles](philosophy.md)** — Restraint, economics, determinism, governance, and discipline as design lenses. -- **[Philosophy of the Site](philosophy.md)** — Five principles: Restraint Over Hype, Economics Over Fashion, Determinism Over Abstraction, Governance Over Chaos, Discipline Over Novelty. +- **[Systems Thinking Glossary](systems-glossary.md)** — Control plane, data plane, abstraction debt, blast radius, data gravity, and related terms. -- **[Systems Thinking Glossary](systems-glossary.md)** — Operationally-oriented definitions for: control plane, data plane, abstraction debt, blast radius, data gravity, eventual consistency, metadata debt, operational entropy, and schema contract. +### New deep dives -- **[ADR 0014: Elevate Site to Systems Doctrine](adr/0014-elevate-site-to-systems-doctrine.md)** — Structural decision to add reading tracks, decision frameworks, anti-patterns, philosophy, and glossary pages to transform the site into a navigable intellectual framework. +- **[Why Most Kubernetes Clusters Shouldn't Exist](deep-dives/why-most-kubernetes-clusters-shouldnt-exist.md)** — Orchestration overhead, etcd fragility, networking complexity, organizational maturity requirements, and the portability illusion. *(Themes: Infrastructure · Architecture · Economics)* -### New Deep Dives +- **[The End of the Data Warehouse?](deep-dives/the-end-of-the-data-warehouse.md)** — Lakehouse convergence, open table formats, DuckDB compute fragmentation, governance implications, and a tiered decision framework. *(Themes: Data Architecture · Economics · Ecosystem)* -- **[Why Most Kubernetes Clusters Shouldn't Exist](deep-dives/why-most-kubernetes-clusters-shouldnt-exist.md)** — Orchestration overhead, etcd fragility, networking complexity, organizational maturity requirements, and the portability illusion. Decision matrix by team size and workload. *(Themes: Infrastructure · Architecture · Economics)* +- **[The Economics of GPU Infrastructure](deep-dives/the-economics-of-gpu-infrastructure.md)** — Utilization patterns, interconnect economics, training vs inference cost structures, and a buy-vs-rent decision matrix. *(Themes: Infrastructure · Economics · ML Systems)* -- **[The End of the Data Warehouse?](deep-dives/the-end-of-the-data-warehouse.md)** — The warehouse is not dying — it is mutating. Lakehouse convergence, open table formats, DuckDB compute fragmentation, governance implications, and a tiered decision framework. *(Themes: Data Architecture · Economics · Ecosystem)* +- **[The Myth of Serverless Simplicity](deep-dives/the-myth-of-serverless-simplicity.md)** — Cold starts, IAM explosion, observability fragmentation, vendor lock-in, and per-invocation economics at scale. *(Themes: Infrastructure · Economics · Architecture)* -- **[The Economics of GPU Infrastructure](deep-dives/the-economics-of-gpu-infrastructure.md)** — GPU scarcity, utilization patterns (~21% realistic vs 100% theoretical), CUDA/driver operational complexity, PCIe vs NVLink interconnect economics, training vs inference cost structures, and a buy-vs-rent decision matrix. *(Themes: Infrastructure · Economics · ML Systems)* +- **[Why Most ML Systems Fail in Production](deep-dives/why-ml-systems-fail-in-production.md)** — Training/serving mismatch, data drift, feature skew, silent degradation, and when heuristics are the right choice. *(Themes: Data Architecture · Organizational · Economics)* -- **[The Myth of Serverless Simplicity](deep-dives/the-myth-of-serverless-simplicity.md)** — Serverless relocates complexity rather than eliminating it. Cold starts, IAM explosion, observability fragmentation, vendor lock-in, and the economics of per-invocation billing at scale. *(Themes: Infrastructure · Economics · Architecture)* +- **[The Physics of Storage Systems](deep-dives/the-physics-of-storage-systems.md)** — HDD through NVMe through object storage, IO amplification, throughput vs IOPS, and a storage tier decision framework. *(Themes: Storage · Infrastructure · Economics)* -- **[Why Most ML Systems Fail in Production](deep-dives/why-ml-systems-fail-in-production.md)** — Training/serving mismatch, data drift, feature skew, silent degradation, organizational misalignment, ML observability beyond accuracy metrics, and when heuristics are the right choice. *(Themes: Data Architecture · Organizational · Economics)* +- **[The Operational Geometry of Spatial Systems](deep-dives/the-operational-geometry-of-spatial-systems.md)** — Quadtree vs R-tree vs H3, COG vs GeoParquet, H3 partitioning, routing graph vs raster cost surfaces. *(Themes: Spatial · Architecture · Data Formats)* -- **[The Physics of Storage Systems](deep-dives/the-physics-of-storage-systems.md)** — HDD rotational latency through NVMe through cloud object storage. IO amplification, throughput vs IOPS, range requests, cold tiers, and a storage tier decision framework by workload type. *(Themes: Storage · Infrastructure · Economics)* - -- **[The Operational Geometry of Spatial Systems](deep-dives/the-operational-geometry-of-spatial-systems.md)** — Quadtree vs R-tree vs H3 hex grid indexing, COG vs GeoParquet format interaction, H3 partitioning strategies, routing graph vs raster cost surface trade-offs. *(Themes: Spatial · Architecture · Data Formats)* - -- **[The Hidden Cost of Metadata Debt](deep-dives/the-hidden-cost-of-metadata-debt.md)** — Catalog drift patterns, control plane collapse, governance vs bureaucracy, the economic cost of duplicate pipelines and compliance failure, and a progressive enforcement framework. *(Themes: Governance · Data Architecture · Economics)* - -- **[ADR 0012–0013: Deep Dive Scale Governance and Curation Model](adr/0012-deep-dives-scale-governance.md)** — Formal curation criteria (novelty, substance, durability, and non-tutorial tests), cluster soft maxima, cross-link requirements, and obsolescence handling for the now-29-essay Deep Dives corpus. - -### ASCII → Mermaid Diagram Conversions - -Converted ASCII diagrams in four deep dives to rendered Mermaid blocks: - -- **Prefect vs Airflow** — Airflow control plane / execution layer; Prefect API vs worker data plane separation -- **ESP32 vs Raspberry Pi** — Side-by-side execution model stacks (bare-metal vs Linux kernel layers) -- **Why Most Data Pipelines Fail** — Monolithic orchestration fan-out; DAG spaghetti coupling; shared warehouse coupling; event-driven chaos -- **Observability vs Monitoring** — Sampling trade-off (100% / tail-1% / head-0.1%); data pipeline observability dimensions (freshness, completeness, correctness, lineage) +- **[The Hidden Cost of Metadata Debt](deep-dives/the-hidden-cost-of-metadata-debt.md)** — Catalog drift, control plane collapse, duplicate pipelines, compliance failure, and progressive enforcement. *(Themes: Governance · Data Architecture · Economics)* ### Recursive Cathedral Generator (Kotlin + Processing) - **[Recursive Cathedral Generator](tutorials/just-for-fun/kotlin-recursive-cathedral.md)** — Grow Gothic cathedral silhouettes from recursive L-system rules. Deterministic seeds, bilateral symmetry, exportable PNG frames. -- **[Pi-Based Sample Library Server with Live Audition](tutorials/just-for-fun/pi-sample-server.md)** — Raspberry Pi sample server with SQLite indexing, Nginx, FastAPI, WebAudio API, and USB MIDI live audition. Full 10-section deep dive. +- **[Pi-Based Sample Library Server with Live Audition](tutorials/just-for-fun/pi-sample-server.md)** — Raspberry Pi sample server with SQLite indexing, Nginx, FastAPI, WebAudio API, and USB MIDI live audition. ## Late 2025 @@ -80,29 +65,29 @@ Converted ASCII diagrams in four deep dives to rendered Mermaid blocks: - **[Fractal Art Explorer (JavaScript)](tutorials/just-for-fun/fractal-art-explorer-js.md)** — Real-time Mandelbrot/Julia explorer with GPU acceleration, custom palettes, and orbit traps. -- **[OSC + MQTT + Prometheus + SuperCollider](tutorials/just-for-fun/osc-mqtt-prometheus-supercollider.md)** — Let your metrics sing. Wire Prometheus exporters into SuperCollider via OSC for live data sonification. +- **[OSC + MQTT + Prometheus + SuperCollider](tutorials/just-for-fun/osc-mqtt-prometheus-supercollider.md)** — Wire Prometheus exporters into SuperCollider via OSC for live data sonification. - **[MCP + FastAPI Full Stack](best-practices/ml-ai/mcp-fastapi-stack.md)** — Model Context Protocol integrated with FastAPI for production LLM toolchains. -- **[Cross-Domain Identity Federation](best-practices/security/identity-federation-authz-authn-architecture.md)** — Unified identity federation architecture across all system layers and environments. +- **[Cross-Domain Identity Federation](best-practices/security/identity-federation-authz-authn-architecture.md)** — Unified identity federation architecture across system layers and environments. - **[Environment Promotion Drift Governance](best-practices/operations-monitoring/environment-promotion-drift-governance.md)** — Controlled environment promotion with drift prevention and release channel management. -- **[Data Quality SLA Validation](best-practices/data-governance/data-quality-sla-validation-observability.md)** — Comprehensive data quality governance with SLAs and multi-layer validation for tabular, geospatial, and ML data. +- **[Data Quality SLA Validation](best-practices/data-governance/data-quality-sla-validation-observability.md)** — Data quality governance with SLAs and multi-layer validation for tabular, geospatial, and ML data. ## Mid 2025 -- **[IAM & RBAC Governance](best-practices/security/iam-rbac-abac-governance.md)** — Identity and access management across heterogeneous stacks. One of the most-linked pages on the site. +- **[IAM & RBAC Governance](best-practices/security/iam-rbac-abac-governance.md)** — Identity and access management across heterogeneous stacks. -- **[Release Management & Progressive Delivery](best-practices/operations-monitoring/release-management-and-progressive-delivery.md)** — Safe deployment strategies for coordinating changes across applications, databases, data pipelines, and ML systems. +- **[Release Management & Progressive Delivery](best-practices/operations-monitoring/release-management-and-progressive-delivery.md)** — Safe deployment strategies across applications, databases, data pipelines, and ML systems. - **[Holistic Capacity Planning](best-practices/architecture-design/capacity-planning-and-workload-modeling.md)** — Workload modeling, scaling economics, and resource prediction frameworks. -- **[ONNX Browser Inference](tutorials/ml-ai/onnx-browser-inference.md)** — Run ML models directly in the browser with ONNX Runtime Web. No server required. +- **[ONNX Browser Inference](tutorials/ml-ai/onnx-browser-inference.md)** — Run ML models in the browser with ONNX Runtime Web. -- **[RKE2 on Raspberry Pi Farm](tutorials/docker-infrastructure/rke2-raspberry-pi.md)** — Full Kubernetes cluster on ARM hardware. Practical, reproducible, and genuinely fun. +- **[RKE2 on Raspberry Pi Farm](tutorials/docker-infrastructure/rke2-raspberry-pi.md)** — Kubernetes cluster on ARM hardware. -- **[PostGIS Geometry Indexing](tutorials/database-data-engineering/postgis-geometry-indexing.md)** — Spatial index strategies, operator classes, and query plans for production PostGIS deployments. +- **[PostGIS Geometry Indexing](tutorials/database-data-engineering/postgis-geometry-indexing.md)** — Spatial index strategies, operator classes, and query plans for production PostGIS. --- diff --git a/mkdocs.yml b/mkdocs.yml index 7ced8b6..c0a1fd5 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,5 +1,5 @@ site_name: Joshua N. Grant - Geospatial Systems Architect -site_description: Professional portfolio and technical documentation for Joshua N. Grant, Geospatial Systems Architect at Oak Ridge National Laboratory +site_description: Technical writing and project notes by Joshua N. Grant, Geospatial Systems Architect at Oak Ridge National Laboratory site_author: Joshua N. Grant site_url: https://sempervent.github.io repo_url: https://github.com/sempervent/sempervent.github.io @@ -11,16 +11,14 @@ theme: # Distinctive teal/cyan palette - professional yet personal # Maintains strong contrast for accessibility palette: - - media: "(prefers-color-scheme: light)" - scheme: default - primary: teal # Deep teal for professional, distinctive look - accent: cyan # Cyan accent for interactive elements + - scheme: default + primary: teal + accent: cyan toggle: icon: material/brightness-7 name: Switch to dark mode - - media: "(prefers-color-scheme: dark)" - scheme: slate - primary: teal # Consistent teal in dark mode + - scheme: slate + primary: teal accent: cyan toggle: icon: material/brightness-4 @@ -59,9 +57,10 @@ theme: view: material/eye # Custom CSS for typography and personalization custom_dir: docs/assets/overrides - # Additional CSS for readability improvements - extra_css: - - assets/styles/custom.css + +# Loaded after Material styles (must be top-level, not under theme) +extra_css: + - assets/styles/custom.css nav: - Home: index.md @@ -71,37 +70,16 @@ nav: - Writing: - Technical overview: documentation.md - Recent additions: whats-new.md - - Tags: tags.md - - Doctrine: - - Start Here — Architectural Compass: start-here-architectural-paths.md + - Start Here: + - Architectural Compass: start-here-architectural-paths.md - Reading Tracks: reading-tracks.md - Decision Frameworks: decision-frameworks.md - Anti-Patterns: anti-patterns.md - Philosophy: philosophy.md - Systems Glossary: systems-glossary.md - - Diagrams: - - Diagram Style Guide: diagrams/style-guide.md - - Architecture Decisions: - - Overview: adr/index.md - - "ADR-0001: Site Architecture": adr/0001-site-architecture.md - - "ADR-0002: MkDocs + Material": adr/0002-why-mkdocs-material.md - - "ADR-0003: Best Practices vs Tutorials": adr/0003-why-best-practices-vs-tutorials.md - - "ADR-0004: Why Just for Fun Exists": adr/0004-why-just-for-fun-exists.md - - "ADR-0005: ESP32 Section Architecture": adr/0005-esp32-section-architecture.md - - "ADR-0006: Embedded Expansion (LoRa + Power)": adr/0006-embedded-expansion-lora-power.md - - "ADR-0007: Deep Dives Section": adr/0007-deep-dives-section.md - - "ADR-0008: Expand Deep Dives Section": adr/0008-expand-deep-dives-section.md - - "ADR-0009: Deep Dives Expansion": adr/0009-deep-dives-expansion.md - - "ADR-0010: Deep Dives Governance": adr/0010-deep-dives-governance.md - - "ADR-0011: Deep Dives Expansion Phase II": adr/0011-deep-dives-expansion-phase-2.md - - "ADR-0012: Deep Dives Scale Governance": adr/0012-deep-dives-scale-governance.md - - "ADR-0013: Deep Dives Curation Model": adr/0013-deep-dives-curation-model.md - - "ADR-0014: Elevate Site to Systems Doctrine": adr/0014-elevate-site-to-systems-doctrine.md - - "ADR-0015: Standardize Diagrams on Mermaid": adr/0015-diagrams-mermaid.md - Best Practices: - Overview: best-practices/index.md - - 📌 Start Here: - - ADR and Technical Decision Governance: best-practices/architecture-design/adr-decision-governance.md + - Start Here: - Configuration Management: best-practices/operations-monitoring/configuration-management.md - System Resilience & Concurrency: best-practices/operations-monitoring/system-resilience-and-concurrency.md - IAM & RBAC Governance: best-practices/security/iam-rbac-abac-governance.md @@ -121,11 +99,11 @@ nav: - Spark on Kubernetes: best-practices/data-processing/spark/spark-on-kubernetes.md - Spark Performance Tuning: best-practices/data-processing/spark/spark-performance-tuning.md - Spark in Modern Architectures: best-practices/data-processing/spark/spark-modern-architecture.md - - 🐍 Python Development: + - Python Development: - Overview: best-practices/python/index.md - "Developer Experience (DX) as Infrastructure: Golden Paths, Tooling Ecosystems & Workflow Automation": best-practices/python/dx-architecture-and-golden-paths.md - Python Package: best-practices/python/python-package.md - - Typing in Python: best-practices/python/typing-in-python.md + - Python type hints: best-practices/python/python-type-hints.md - Python Concurrency: best-practices/python/python-threading-and-multiprocessing.md - Python Async Best Practices: best-practices/python/python-async-best-practices.md - Pytest Best Practices: best-practices/python/pytest-best-practices.md @@ -133,18 +111,18 @@ nav: - FastAPI Geospatial: best-practices/python/fastapi-geospatial.md - Web Performance Optimization: best-practices/python/web-performance-optimization.md - TUI Applications: best-practices/python/tui-applications.md - - 🦀 Rust Development: + - Rust Development: - Overview: best-practices/rust/index.md - Rust Development Environment: best-practices/rust/rust-dev-environment.md - TUI Applications: best-practices/rust/tui-applications.md - - 🐹 Go Development: + - Go Development: - Overview: best-practices/go/index.md - Go Development Environment: best-practices/go/go-dev-environment.md - TUI Applications: best-practices/go/tui-applications.md - - 📊 R Development: + - R Development: - Overview: best-practices/r/index.md - R Development Environment: best-practices/r/r-dev-environment.md - - 🐳 Docker & Infrastructure: + - Docker & Infrastructure: - Overview: best-practices/docker-infrastructure/index.md - Docker & Compose: best-practices/docker-infrastructure/docker-and-compose.md - Conda to Docker Migration: best-practices/docker-infrastructure/conda-to-docker-migration.md @@ -157,12 +135,12 @@ nav: - Ansible Performance Optimization: best-practices/docker-infrastructure/ansible-performance-optimization.md - Jinja Best Practices: best-practices/docker-infrastructure/jinja-best-practices.md - Advanced Tmux Workflows: best-practices/docker-infrastructure/tmux-advanced.md - - 📝 Git & Version Control: + - Git & Version Control: - Overview: best-practices/git/index.md - Git Production: best-practices/git/git-production.md - Git Workflows & Collaboration: best-practices/git/git-workflows-collaboration.md - Modern GitFlow Best Practices: best-practices/git/gitflow-best-practices.md - - 🗄️ Database & Data Management: + - Database & Data Management: - Overview: best-practices/database-data/index.md - "Cross-System Data Lineage, Inter-Service Metadata Contracts & Provenance Enforcement": best-practices/database-data/data-lineage-contracts.md - Semantic Layer Engineering, Domain Models, and Knowledge Graph Alignment: best-practices/database-data/semantic-layer-engineering.md @@ -178,7 +156,7 @@ nav: - Data Lake Governance: best-practices/database-data/data-lake-governance.md - Geospatial Data Engineering: best-practices/database-data/geospatial-data-engineering.md - Geospatial Benchmarking: best-practices/database-data/geospatial-benchmarking.md - - 🐘 PostgreSQL Development: + - PostgreSQL Development: - Overview: best-practices/postgres/index.md - Core & Design: - Development Environment: best-practices/postgres/postgres-dev-environment.md @@ -216,14 +194,14 @@ nav: - Deployment Strategies: best-practices/postgres/postgres-deployment-strategies.md - Data Engineering: best-practices/database-data/data-engineering.md - ETL Pipeline Design: best-practices/database-data/etl-pipeline-design.md - - 📊 Data Governance: + - Data Governance: - Overview: best-practices/data-governance/index.md - Metadata Standards, Schema Governance & Data Provenance: best-practices/data-governance/metadata-provenance-contracts.md - Data Validation and Contract Governance: best-practices/data-governance/data-validation-and-contract-governance.md - Data Freshness, SLA/SLO Governance, and Pipeline Reliability Contracts: best-practices/data-governance/data-freshness-sla-governance.md - Data Retention, Archival Strategy, Lifecycle Governance & Cold Storage Patterns: best-practices/data-governance/data-retention-archival-lifecycle-governance.md - Data Quality SLAs, Validation Layers, and Observability for Tabular, Geospatial, and ML Data: best-practices/data-governance/data-quality-sla-validation-observability.md - - 🤖 Machine Learning & AI: + - Machine Learning & AI: - Overview: best-practices/ml-ai/index.md - "ML Systems Architecture: Feature Stores, Model Serving, Experiment Governance, and Cross-System Reproducibility": best-practices/ml-ai/ml-systems-architecture-governance.md - Prompting LLMs: best-practices/ml-ai/prompting-llms.md @@ -232,7 +210,7 @@ nav: - Embeddings & Vector Databases: best-practices/ml-ai/embeddings-and-vector-databases.md - Vibe → Agentic LLMs: best-practices/ml-ai/vibe-to-agentic.md - R Data Exploration: best-practices/ml-ai/r-data-exploration.md - - 🏗️ Architecture & Design: + - Architecture & Design: - Overview: best-practices/architecture-design/index.md - System-Wide Naming, Taxonomy, and Structural Vocabulary Governance: best-practices/architecture-design/system-taxonomy-governance.md - Cost-Aware Architecture & Resource-Efficiency Governance: best-practices/architecture-design/cost-aware-architecture-and-efficiency-governance.md @@ -252,7 +230,6 @@ nav: - Secrets Management: best-practices/architecture-design/secrets-management.md - Testing & CI/CD Pipelines: best-practices/architecture-design/ci-cd-pipelines.md - Documentation: best-practices/architecture-design/documentation.md - - ADR and Technical Decision Governance: best-practices/architecture-design/adr-decision-governance.md - Caching & Performance Layers: best-practices/architecture-design/caching-performance.md - Cache-Topology Architecture: best-practices/architecture-design/cache-topology-architecture.md - Data Mesh Architecture: best-practices/architecture-design/data-mesh-architecture.md @@ -261,7 +238,7 @@ nav: - Reference Architecture Diagrams: best-practices/architecture-design/reference-architecture-diagrams.md - RDF/OWL Metadata Automation: best-practices/architecture-design/rdf-owl-metadata-automation.md - Protocol Buffers with Python: best-practices/architecture-design/protobuf-python.md - - 🔧 Operations & Monitoring: + - Operations & Monitoring: - Overview: best-practices/operations-monitoring/index.md - "Observability as Architecture: Unified Telemetry Models Across Clusters, Services, and Languages": best-practices/operations-monitoring/unified-observability-architecture.md - Chaos Engineering, Fault Injection, and Reliability Validation: best-practices/operations-monitoring/chaos-engineering-governance.md @@ -279,7 +256,7 @@ nav: - Testing Best Practices: best-practices/operations-monitoring/testing-best-practices.md - Secrets & Configuration Management: best-practices/operations-monitoring/secrets-config.md - Grafana: best-practices/operations-monitoring/grafana.md - - 🔒 Security: + - Security: - Overview: best-practices/security/index.md - End-to-End Secrets Management & Key Rotation: best-practices/security/secrets-governance.md - Identity & Access Management, RBAC/ABAC, and Least-Privilege Governance: best-practices/security/iam-rbac-abac-governance.md @@ -287,23 +264,23 @@ nav: - Secure Computes, Sandboxing, and Multi-Tenant Isolation for Polyglot Systems: best-practices/security/secure-sandboxing-and-multi-tenant-isolation.md - Cross-Domain Identity Federation, AuthZ/AuthN Architecture & Identity Propagation Models: best-practices/security/identity-federation-authz-authn-architecture.md - Secret Supply Chains, Encryption Lifecycle Management & Cryptographic Rotation Strategy: best-practices/security/encryption-lifecycle-and-crypto-rotation.md - - ⚡ Performance: + - Performance: - Overview: best-practices/performance/index.md - End-to-End Caching Strategy: best-practices/performance/end-to-end-caching-strategy.md - - 🧪 Testing: + - Testing: - Overview: best-practices/testing/index.md - End-to-End Testing Strategy: best-practices/testing/end-to-end-testing-strategy.md - - 🖼️ Diagrams: + - Diagrams: - Systems Diagramming Best Practices: best-practices/diagrams/systems-diagramming-best-practices.md - SVG Workflow Generation: best-practices/diagrams/svg-workflow-generation.md - - 🎨 Creative & Fun: + - Creative & Fun: - Overview: best-practices/creative-fun/index.md - Time Hygiene (UTC, TZ, Clocks): best-practices/creative-fun/time-hygiene.md - Idempotency & De-dup: best-practices/creative-fun/idempotency-and-dedup.md - Celery Tasks: best-practices/creative-fun/celery-best-practices.md - LaTeX Workflows: best-practices/creative-fun/latex.md - YAML Recipe Format: best-practices/creative-fun/yaml-recipe-format.md - - 🔌 Embedded Systems & ESP32: + - Embedded Systems & ESP32: - Overview: best-practices/esp32/index.md - Programming Architecture: best-practices/esp32/esp32-programming-architecture.md - Power Management & Deep Sleep: best-practices/esp32/power-management-and-deep-sleep.md @@ -315,21 +292,21 @@ nav: - LoRa Best Practices (SX127x): best-practices/esp32/lora-best-practices-sx127x.md - Safety Checklist (Printable): best-practices/esp32/esp32-safety-checklist-printable.md - ESP32-S3 and C3 Notes: best-practices/esp32/esp32-s3-and-c3-architecture-notes.md - - 🔋 Power Electronics & Embedded Hardware: + - Power Electronics & Embedded Hardware: - Overview: best-practices/embedded/index.md - Power Electronics for ESP32: best-practices/embedded/power-electronics-for-esp32.md - - 🏠 Home Automation & MQTT: + - Home Automation & MQTT: - Overview: best-practices/home-automation/index.md - Home Assistant Security: best-practices/home-automation/home-assistant-security-best-practices.md - Deep Dives: - Overview: deep-dives/index.md - - 📦 Data Formats & Storage: + - Data Formats & Storage: - Parquet vs CSV vs ORC vs Avro: deep-dives/parquet-vs-csv-orc-avro.md - Geospatial File Format Choices: deep-dives/geospatial-file-format-choices.md - Polars vs Pandas for Geospatial Data: deep-dives/polars-vs-pandas-geospatial.md - The Physics of Storage Systems: deep-dives/the-physics-of-storage-systems.md - The Operational Geometry of Spatial Systems: deep-dives/the-operational-geometry-of-spatial-systems.md - - 🧩 Systems Design & Architecture: + - Systems Design & Architecture: - Why Most Microservices Should Be Monoliths: deep-dives/why-most-microservices-should-be-monoliths.md - Appropriate Use of Microservices: deep-dives/appropriate-use-of-microservices.md - Distributed Systems and the Myth of Infinite Scale: deep-dives/distributed-systems-myth-of-infinite-scale.md @@ -341,11 +318,11 @@ nav: - Blockchain vs Hashchain: deep-dives/blockchain-vs-hashchain.md - Merkle Trees Explained: deep-dives/merkle-trees-explained.md - Proof of Work Explained: deep-dives/proof-of-work-explained.md - - 🔭 Operations & Reliability: + - Operations & Reliability: - Observability vs Monitoring: deep-dives/observability-vs-monitoring.md - The Economics of Observability: deep-dives/the-economics-of-observability.md - Designing Resilient Distributed Systems: deep-dives/resilient-distributed-systems.md - - 🐳 Infrastructure & Automation: + - Infrastructure & Automation: - Container Base Image Philosophy: deep-dives/container-base-image-philosophy.md - IaC vs GitOps: deep-dives/iac-vs-gitops.md - The Human Cost of Automation: deep-dives/the-human-cost-of-automation.md @@ -353,7 +330,7 @@ nav: - Building a Bitcoin Mining Rig: deep-dives/building-a-bitcoin-mining-rig.md - Converting Bitcoin Mining to LLM Clusters: deep-dives/converting-bitcoin-mining-to-llm.md - The Myth of Serverless Simplicity: deep-dives/the-myth-of-serverless-simplicity.md - - 🏗️ Data Systems & Architecture: + - Data Systems & Architecture: - Lakehouse vs Warehouse vs Database: deep-dives/lakehouse-vs-warehouse-vs-database.md - DuckDB vs PostgreSQL vs Spark: deep-dives/duckdb-vs-postgres-vs-spark.md - Why Most Data Pipelines Fail: deep-dives/why-most-data-pipelines-fail.md @@ -371,17 +348,17 @@ nav: - Why Most Data Lakes Become Data Swamps: deep-dives/data-lakes-become-data-swamps.md - The Metadata Crisis in Modern Data Platforms: deep-dives/metadata-crisis-in-modern-data-platforms.md - Why Most Data Pipelines Are Operationally Fragile: deep-dives/why-data-pipelines-are-operationally-fragile.md - - 📡 Embedded & Radio: + - Embedded & Radio: - LoRaWAN vs Raw LoRa: deep-dives/lorawan-vs-raw-lora.md - MQTT vs HTTP in IoT Systems: deep-dives/mqtt-vs-http-iot.md - ESP32 vs Raspberry Pi: deep-dives/esp32-vs-raspberry-pi.md - Tutorials: - Overview: tutorials/index.md - - 🚀 Quick Start: + - Quick Start: - Overview: tutorials/quick-start/index.md - Creating MkDocs GitHub Site: tutorials/quick-start/creating-mkdocs-github-site.md - Monitoring with Grafana & Prometheus: tutorials/quick-start/monitoring-with-grafana-prometheus.md - - 🐍 Python Development: + - Python Development: - Overview: tutorials/python-development/index.md - psycopg2 to psycopg 3 Migration: tutorials/python-development/psycopg2-to-psycopg3-migration.md - Ruff Check Ignore in pyproject.toml: tutorials/python-development/ruff-check-ignore-pyproject.md @@ -393,15 +370,15 @@ nav: - WebSocket Chat with FastAPI: tutorials/python-development/websocket-chat-fastapi.md - Chaos Engineering with Kubernetes and Python: tutorials/python-development/chaos-engineering-k8s-python.md - Building a Python TUI: tutorials/python-development/building-a-python-tui.md - - 🦀 Rust Development: + - Rust Development: - Overview: tutorials/rust-development/index.md - Rust + CSR (Parse & Build from Parquet/DB): tutorials/rust-development/rust-csr-parquet-db.md - Event-Sourcing in Rust: tutorials/rust-development/rust-event-sourcing.md - Building a Rust TUI: tutorials/rust-development/building-a-rust-tui.md - - 🐹 Go Development: + - Go Development: - Overview: tutorials/go-development/index.md - Building a Go TUI: tutorials/go-development/building-a-go-tui.md - - 🐳 Docker & Infrastructure: + - Docker & Infrastructure: - Overview: tutorials/docker-infrastructure/index.md - Multi-Stage Docker (Conda → scratch): tutorials/docker-infrastructure/multistage-conda-to-scratch.md - Slim Geospatial + GPU Containers (GDAL): tutorials/docker-infrastructure/slim-geospatial-gpu-conda.md @@ -415,7 +392,7 @@ nav: - Deploy SLURM with Ansible on Raspberry Pi Cluster: tutorials/docker-infrastructure/ansible-slurm-raspberrypi.md - Harbor Container Registry Setup: tutorials/docker-infrastructure/harbor-registry-setup.md - Dask with Ansible (CPU + GPU): tutorials/docker-infrastructure/ansible-dask-heterogeneous.md - - 🗄️ Database & Data Engineering: + - Database & Data Engineering: - Overview: tutorials/database-data-engineering/index.md - PostGIS Geometry Indexing: tutorials/database-data-engineering/postgis-geometry-indexing.md - PostGIS Raster Indexing: tutorials/database-data-engineering/postgis-raster-indexing.md @@ -440,7 +417,7 @@ nav: - Graph vs Vector Databases: tutorials/database-data-engineering/graph-vs-vector-databases.md - Geospatial Knowledge Graph: tutorials/database-data-engineering/geospatial-knowledge-graph.md - DuckDB Parquet Data Quality: tutorials/database-data-engineering/duckdb-parquet-data-quality.md - - 🤖 Machine Learning & AI: + - Machine Learning & AI: - Overview: tutorials/ml-ai/index.md - MLflow API Experiments: tutorials/ml-ai/mlflow-api-experiments.md - Local LLM Deployments (Ollama, llama.cpp, vLLM, TGI): tutorials/ml-ai/local-llm-deployments.md @@ -448,22 +425,22 @@ nav: - RAG with Ollama + Database: tutorials/ml-ai/rag-ollama-db.md - MCP ↔ MLflow Toolchain: tutorials/ml-ai/mcp-mlflow-toolchain.md - Semantic ML Training: tutorials/ml-ai/semantic-ml-training.md - - 🔧 System Administration: + - System Administration: - Overview: tutorials/system-administration/index.md - iPXE Multi-System Booting: tutorials/system-administration/ipxe-multi-boot.md - Remote Dev with tmux & screen: tutorials/system-administration/remote-dev-tmux-screen.md - FIFO Prefect Flow with Redis: tutorials/system-administration/prefect-fifo-redis.md - AWK Unix Text Processing: tutorials/system-administration/awk-unix-text-processing.md - - 🖼️ Diagrams: + - Diagrams: - "Layered Systems Diagrams: Mermaid → SVG": tutorials/diagrams/layered-systems-diagrams-mermaid-to-svg.md - - Mermaid → SVG Workflow Pipeline: tutorials/diagrams/mermaid-to-svg-workflow-pipeline.md - - 📊 Data Science & Visualization: + - Rendering Mermaid diagrams to SVG: tutorials/diagrams/mermaid-to-svg-workflow-pipeline.md + - Data Science & Visualization: - Overview: tutorials/data-science-visualization/index.md - Jupyter Notebook Best Practices (Geospatial Edition): tutorials/data-science-visualization/jupyter-notebook-best-practices-geo.md - Mermaid Diagrams in MkDocs: tutorials/data-science-visualization/mermaid-diagrams.md - TikZ Diagrams in LaTeX: tutorials/data-science-visualization/latex-tikz-diagrams.md - Generative Art in R: tutorials/data-science-visualization/r-generative-art.md - - 🛠️ Development Tools: + - Development Tools: - Overview: tutorials/development-tools/index.md - jq JSON Parsing Mastery: tutorials/development-tools/jq-json-parsing-mastery.md - find_files for parquet_s3_fdw: tutorials/development-tools/find-files-parquet-fdw.md @@ -472,7 +449,7 @@ nav: - Python Modbus Device Communication: tutorials/development-tools/python-modbus-devices.md - Mixing Tech with Go Glue: tutorials/development-tools/go-tech-mixer.md - Tauri + rqlite + Syncthing: tutorials/development-tools/tauri-rqlite-syncthing.md - - 🎨 Just for Fun: + - Just for Fun: - Overview: tutorials/just-for-fun/index.md - Glitch Observatory (JS): tutorials/just-for-fun/js-glitch-observatory.md - Terminal to GIF: tutorials/just-for-fun/terminal-to-gif.md @@ -494,7 +471,7 @@ nav: - MIDI-Driven Particle Nebula (Kotlin): tutorials/just-for-fun/kotlin-midi-particle-nebula.md - Cellular Automata Garden (Kotlin): tutorials/just-for-fun/kotlin-cellular-automata-garden.md - Pi Infinite Art Frame (Kotlin): tutorials/just-for-fun/pi-infinite-art-frame-kotlin.md - - 🔌 Embedded Systems: + - Embedded Systems: - Overview: tutorials/embedded/index.md - ESP32 E-Ink Environmental Monitor: tutorials/embedded/esp32-eink-sensor-monitor.md - ESP32 RF Room Light Controller: tutorials/embedded/esp32-rf-room-light-controller.md @@ -502,7 +479,7 @@ nav: - About: - Professional Profile: about.md - - Contact & Collaboration: getting-started.md + - Contact: getting-started.md markdown_extensions: - pymdownx.snippets: @@ -554,18 +531,45 @@ plugins: - search: lang: en separator: '[\s\-\_\.]+' - - tags - redirects: redirect_maps: - projects.md: projects/index.md + adr/index.md: documentation.md + best-practices/architecture-design/adr-decision-governance.md: best-practices/architecture-design/index.md + tags.md: documentation.md lab/index.md: tutorials/just-for-fun/index.md tutorials/python-development/js-glitch-observatory.md: tutorials/just-for-fun/js-glitch-observatory.md + best-practices/python/typing-in-python.md: best-practices/python/python-type-hints.md + best-practices/python/python-async-programming.md: best-practices/python/python-async-best-practices.md + best-practices/python/python-concurrency-patterns.md: best-practices/python/python-threading-and-multiprocessing.md + best-practices/python/python-package-development.md: best-practices/python/python-package.md + tutorials/best-practices-integration/api-first-polyglot-protobuf.md: tutorials/best-practices-integration/index.md + tutorials/best-practices-integration/cost-optimized-geokg-capacity-planning.md: tutorials/best-practices-integration/index.md + tutorials/best-practices-integration/cost-optimized-ml-pipeline-capacity-retention.md: tutorials/best-practices-integration/index.md + tutorials/best-practices-integration/data-pipeline-quality-governance-observability.md: tutorials/best-practices-integration/index.md + tutorials/best-practices-integration/developer-experience-taxonomy-repository.md: tutorials/best-practices-integration/index.md + tutorials/best-practices-integration/documentation-driven-adr-governance.md: tutorials/best-practices-integration/index.md + tutorials/best-practices-integration/event-driven-geokg-observability.md: tutorials/best-practices-integration/index.md + tutorials/best-practices-integration/event-driven-microservices-observability-data-governance.md: tutorials/best-practices-integration/index.md + tutorials/best-practices-integration/geospatial-data-mesh-cost-capacity.md: tutorials/best-practices-integration/index.md + tutorials/best-practices-integration/high-performance-caching-topology.md: tutorials/best-practices-integration/index.md + tutorials/best-practices-integration/high-performance-geokg-async-caching.md: tutorials/best-practices-integration/index.md + tutorials/best-practices-integration/multi-environment-deployment-secrets-config-governance.md: tutorials/best-practices-integration/index.md + tutorials/best-practices-integration/multi-region-dr-temporal-blast-radius.md: tutorials/best-practices-integration/index.md + tutorials/best-practices-integration/multi-region-geokg-temporal-dr.md: tutorials/best-practices-integration/index.md + tutorials/best-practices-integration/polyglot-streaming-saga-cqrs-multicloud.md: tutorials/best-practices-integration/index.md + tutorials/best-practices-integration/progressive-delivery-release-fitness.md: tutorials/best-practices-integration/index.md + tutorials/best-practices-integration/quality-governed-geokg-retention-lifecycle.md: tutorials/best-practices-integration/index.md + tutorials/best-practices-integration/resilient-microservices-chaos-engineering.md: tutorials/best-practices-integration/index.md + tutorials/best-practices-integration/secure-polyglot-identity-encryption.md: tutorials/best-practices-integration/index.md + tutorials/best-practices-integration/semantic-knowledge-graph-rdf-lineage.md: tutorials/best-practices-integration/index.md - exclude: glob: - "_generated/**" + - "archive/**" - git-revision-date-localized: enable_creation_date: true enable_git_follow: false + fallback_to_build_date: true type: timeago - minify: minify_html: true @@ -587,13 +591,6 @@ extra: version: provider: mike generator: false - consent: - title: Cookie consent - description: >- - This site uses cookies to enhance your browsing experience and analyze site traffic. - By continuing to use this site, you consent to our use of cookies. - necessary: true - strict: true copyright: | - Copyright © 2025 Joshua N. Grant - + Personal site — not an ORNL publication. Copyright © 2025 Joshua N. Grant — MIT License \ No newline at end of file diff --git a/scripts/lint_writing.py b/scripts/lint_writing.py new file mode 100644 index 0000000..8bd06fb --- /dev/null +++ b/scripts/lint_writing.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Fail on egregious generated-prose fingerprints in technical corpus.""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +DIRS = [ROOT / "docs" / "best-practices", ROOT / "docs" / "tutorials"] + +# Hard failures (serious template voice) +EGREGIOUS = [ + re.compile(r"weapon of choice", re.I), + re.compile(r"complete machinery", re.I), + re.compile(r"Objective:\s*Master", re.I), +] + +# Allowed in Just for Fun only +JFF = "just-for-fun" + + +def main() -> int: + failures: list[str] = [] + for base in DIRS: + for path in sorted(base.rglob("*.md")): + rel = path.relative_to(ROOT / "docs").as_posix() + if JFF in rel: + continue + text = path.read_text(encoding="utf-8", errors="replace") + for pat in EGREGIOUS: + if pat.search(text): + failures.append(f"{rel}: {pat.pattern}") + if failures: + print("Writing lint failures:", file=sys.stderr) + for line in failures[:50]: + print(f" {line}", file=sys.stderr) + if len(failures) > 50: + print(f" ... and {len(failures) - 50} more", file=sys.stderr) + return 1 + print("OK: no egregious writing fingerprints") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/rebuild_nav.py b/scripts/rebuild_nav.py index e85e74d..f741125 100644 --- a/scripts/rebuild_nav.py +++ b/scripts/rebuild_nav.py @@ -9,6 +9,40 @@ ROOT = Path(__file__).resolve().parents[1] MKDOCS = ROOT / "mkdocs.yml" +# Nav sections removed from the public site (skip when merging from main). +_REMOVED_SECTION_HEADERS = frozenset( + { + "- Architecture Decisions:", + "- Diagrams:", + } +) +_SKIP_LINE_SUBSTRINGS = ( + "Contact & Collaboration", + "adr/", + "ADR-", + "Diagram Style Guide", + "adr-decision-governance", +) + + +def _filter_nav_body(lines: list[str]) -> list[str]: + filtered: list[str] = [] + skip_depth: int | None = None + for line in lines: + stripped = line.strip() + if stripped in _REMOVED_SECTION_HEADERS: + skip_depth = len(line) - len(line.lstrip()) + continue + if skip_depth is not None: + if stripped and (len(line) - len(line.lstrip())) <= skip_depth: + skip_depth = None + else: + continue + if any(s in line for s in _SKIP_LINE_SUBSTRINGS): + continue + filtered.append(line) + return filtered + def main() -> None: original = subprocess.check_output( @@ -21,8 +55,7 @@ def main() -> None: nav_lines = lines[nav_start:ext_start] idx = next(i for i, l in enumerate(nav_lines) if l.strip().startswith("- Doctrine:")) - body = nav_lines[idx:] - body = [l for l in body if "Contact & Collaboration" not in l] + body = _filter_nav_body(nav_lines[idx:]) indented: list[str] = [] for line in body: @@ -39,14 +72,13 @@ def main() -> None: - Writing: - Technical overview: documentation.md - What's New: whats-new.md - - Tags: tags.md """ footer = """ - Lab: - Overview: lab/index.md - Just for Fun: tutorials/just-for-fun/index.md - About: - Professional Profile: about.md - - Contact & Collaboration: getting-started.md + - Contact: getting-started.md """ new_content = ( diff --git a/scripts/redirect_url.py b/scripts/redirect_url.py new file mode 100644 index 0000000..ba61606 --- /dev/null +++ b/scripts/redirect_url.py @@ -0,0 +1,16 @@ +"""Map MkDocs doc paths to public URL paths (directory URLs enabled).""" + +from __future__ import annotations + + +def doc_path_to_url(doc_path: str) -> str: + """Convert a docs-relative markdown path to the site URL path.""" + path = doc_path.strip().replace("\\", "/") + if not path.endswith(".md"): + raise ValueError(f"expected .md path, got {doc_path!r}") + path = path[: -len(".md")] + if path.endswith("/index"): + path = path[: -len("/index")] + if not path: + return "/" + return f"/{path}/" diff --git a/tests/test_redirect_maps.py b/tests/test_redirect_maps.py new file mode 100644 index 0000000..4374ea2 --- /dev/null +++ b/tests/test_redirect_maps.py @@ -0,0 +1,119 @@ +"""Redirect map sanity checks and HTTP smoke tests against built site.""" + +from __future__ import annotations + +import re +import subprocess +import sys +import threading +from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from urllib.request import urlopen + +ROOT = Path(__file__).resolve().parents[1] +SITE = ROOT / "site" + +REDIRECT_LINE = re.compile( + r"^\s+(?P[A-Za-z0-9_./-]+\.md):\s*(?P[A-Za-z0-9_./-]+\.md)\s*$", + re.MULTILINE, +) + + +def _load_redirect_maps() -> dict[str, str]: + text = (ROOT / "mkdocs.yml").read_text(encoding="utf-8") + maps: dict[str, str] = {} + for match in REDIRECT_LINE.finditer(text): + maps[match.group("src")] = match.group("dst") + assert maps, "redirect_maps not found in mkdocs.yml" + return maps + + +def test_redirect_maps_have_distinct_source_and_target_urls() -> None: + sys.path.insert(0, str(ROOT / "scripts")) + from redirect_url import doc_path_to_url + + maps = _load_redirect_maps() + assert maps, "expected at least one redirect" + collisions: list[str] = [] + for src, dst in maps.items(): + src_url = doc_path_to_url(src) + dst_url = doc_path_to_url(dst) + if src_url == dst_url: + collisions.append(f"{src} -> {dst} both resolve to {src_url}") + assert not collisions, "self-redirects in mkdocs.yml:\n" + "\n".join(collisions) + + +def test_projects_index_is_not_meta_redirect() -> None: + subprocess.run( + [sys.executable, str(ROOT / "scripts" / "generate_portfolio.py")], + cwd=ROOT, + check=True, + capture_output=True, + ) + mkdocs = ROOT / ".venv" / "bin" / "mkdocs" + subprocess.run( + [str(mkdocs) if mkdocs.exists() else "mkdocs", "build", "--strict"], + cwd=ROOT, + check=True, + capture_output=True, + ) + html = (SITE / "projects" / "index.html").read_text(encoding="utf-8") + assert "http-equiv=refresh" not in html.lower() + assert 'url=/projects/' not in html.lower().replace(" ", "") + + +def _serve_site() -> tuple[ThreadingHTTPServer, str]: + class Handler(SimpleHTTPRequestHandler): + def __init__(self, *args, **kwargs): + super().__init__(*args, directory=str(SITE), **kwargs) + + def log_message(self, *_args): + return + + httpd = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=httpd.serve_forever, daemon=True).start() + base = f"http://127.0.0.1:{httpd.server_address[1]}" + return httpd, base + + +def _get_html(base: str, path: str) -> str: + with urlopen(f"{base}{path}", timeout=10) as resp: + return resp.read().decode("utf-8", errors="replace") + + +def _is_mkdocs_redirect_page(html: str) -> bool: + return "Redirecting..." in html or "http-equiv=refresh" in html.lower() + + +def test_http_routing_smoke() -> None: + if not SITE.exists(): + subprocess.run( + [str(ROOT / ".venv" / "bin" / "mkdocs"), "build", "--strict"], + cwd=ROOT, + check=True, + capture_output=True, + ) + httpd, base = _serve_site() + try: + projects = _get_html(base, "/projects/") + assert not _is_mkdocs_redirect_page(projects), "/projects/ must not be a redirect stub" + assert "project-card" in projects or "Portfolio" in projects + + doc_sites = _get_html(base, "/projects/documentation-sites/") + assert not _is_mkdocs_redirect_page(doc_sites) + + tags = _get_html(base, "/tags/") + assert _is_mkdocs_redirect_page(tags), "/tags/ should redirect once to Writing" + assert "../documentation/" in tags or "/documentation/" in tags + writing = _get_html(base, "/documentation/") + assert not _is_mkdocs_redirect_page(writing) + + glitch = _get_html(base, "/tutorials/python-development/js-glitch-observatory/") + assert _is_mkdocs_redirect_page(glitch) + assert "just-for-fun/js-glitch-observatory" in glitch + + lab = _get_html(base, "/lab/") + assert _is_mkdocs_redirect_page(lab) + assert "just-for-fun" in lab + finally: + httpd.shutdown() diff --git a/tests/test_theme_palette.py b/tests/test_theme_palette.py new file mode 100644 index 0000000..a05535d --- /dev/null +++ b/tests/test_theme_palette.py @@ -0,0 +1,147 @@ +"""Browser checks for Material light/dark palette and project-card computed colors.""" + +from __future__ import annotations + +import subprocess +import sys +import threading +import time +from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +import pytest + +pytest.importorskip("playwright") +from playwright.sync_api import sync_playwright + +ROOT = Path(__file__).resolve().parents[1] +SITE = ROOT / "site" +MKDOCS = ROOT / ".venv" / "bin" / "mkdocs" + + +def _build_site() -> None: + subprocess.run( + [sys.executable, str(ROOT / "scripts" / "validate_projects.py")], + cwd=ROOT, + check=True, + capture_output=True, + ) + subprocess.run( + [sys.executable, str(ROOT / "scripts" / "generate_portfolio.py")], + cwd=ROOT, + check=True, + capture_output=True, + ) + cmd = [str(MKDOCS) if MKDOCS.exists() else "mkdocs", "build", "--strict"] + subprocess.run(cmd, cwd=ROOT, check=True, capture_output=True) + + +def _serve_site() -> tuple[ThreadingHTTPServer, str]: + class Handler(SimpleHTTPRequestHandler): + def __init__(self, *args, **kwargs): + super().__init__(*args, directory=str(SITE), **kwargs) + + def log_message(self, *_args): + return + + httpd = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=httpd.serve_forever, daemon=True).start() + base = f"http://127.0.0.1:{httpd.server_address[1]}/" + return httpd, base + + +def _relative_luminance(rgb: str) -> float: + parts = [float(x) for x in rgb.replace("rgba(", "").replace("rgb(", "").replace(")", "").split(",")[:3]] + r, g, b = [x / 255 for x in parts] + + def lin(v: float) -> float: + return v / 12.92 if v <= 0.03928 else ((v + 0.055) / 1.055) ** 2.4 + + rs, gs, bs = lin(r), lin(g), lin(b) + return 0.2126 * rs + 0.7152 * gs + 0.0722 * bs + + +def _contrast_ratio(fg: str, bg: str) -> float: + l1, l2 = _relative_luminance(fg), _relative_luminance(bg) + lighter, darker = max(l1, l2), min(l1, l2) + return (lighter + 0.05) / (darker + 0.05) + + +def _card_metrics(page) -> dict[str, str]: + page.locator(".project-card-grid .project-card").first.wait_for(timeout=10_000) + return page.evaluate( + """() => { + const card = document.querySelector('.project-card-grid .project-card'); + const body = document.body; + const cs = getComputedStyle(card); + const bs = getComputedStyle(body); + return { + cardBg: cs.backgroundColor, + cardFg: cs.color, + bodyBg: bs.backgroundColor, + scheme: body.getAttribute('data-md-color-scheme') + || document.documentElement.getAttribute('data-md-color-scheme'), + }; + }""" + ) + + +def _assert_light_card(metrics: dict[str, str], where: str) -> None: + bg_l = _relative_luminance(metrics["cardBg"]) + fg_l = _relative_luminance(metrics["cardFg"]) + assert bg_l > 0.85, f"{where}: expected light card bg, got {metrics['cardBg']}" + assert fg_l < 0.4, f"{where}: expected dark card text, got {metrics['cardFg']}" + assert _contrast_ratio(metrics["cardFg"], metrics["cardBg"]) >= 4.5, metrics + + +def _assert_dark_card(metrics: dict[str, str], where: str) -> None: + bg_l = _relative_luminance(metrics["cardBg"]) + fg_l = _relative_luminance(metrics["cardFg"]) + assert bg_l < 0.25, f"{where}: expected dark card bg, got {metrics['cardBg']}" + assert fg_l > 0.6, f"{where}: expected light card text, got {metrics['cardFg']}" + assert _contrast_ratio(metrics["cardFg"], metrics["cardBg"]) >= 4.5, metrics + + +def _click_palette(page, palette_id: str) -> None: + page.locator(f"label[for='{palette_id}']").click(timeout=5000) + page.wait_for_timeout(350) + + +@pytest.fixture(scope="module") +def site_server(): + _build_site() + httpd, base = _serve_site() + yield base + httpd.shutdown() + + +def test_theme_project_cards_home_and_projects(site_server: str) -> None: + base = site_server + with sync_playwright() as p: + browser = p.chromium.launch() + page = browser.new_page() + + # Light (default) + page.goto(base, wait_until="load") + _assert_light_card(_card_metrics(page), "home light") + + # Dark + _click_palette(page, "__palette_1") + _assert_dark_card(_card_metrics(page), "home dark") + + # Reload preserves dark + page.reload(wait_until="load") + _assert_dark_card(_card_metrics(page), "home dark after reload") + + # Projects via direct URL (avoid instant-nav loop on static server) + page.goto(f"{base}projects/", wait_until="load") + _assert_dark_card(_card_metrics(page), "projects dark") + + # Back to light + _click_palette(page, "__palette_0") + _assert_light_card(_card_metrics(page), "projects light") + + page.goto(base, wait_until="load") + _assert_light_card(_card_metrics(page), "home light again") + + browser.close() diff --git a/tests/test_writing_lint.py b/tests/test_writing_lint.py new file mode 100644 index 0000000..e9615f4 --- /dev/null +++ b/tests/test_writing_lint.py @@ -0,0 +1,26 @@ +"""Writing quality guardrails.""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def test_lint_writing_script_exits_zero() -> None: + result = subprocess.run( + [sys.executable, str(ROOT / "scripts" / "lint_writing.py")], + cwd=ROOT, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr or result.stdout + + +def test_redirect_maps_include_merged_python_guides() -> None: + text = (ROOT / "mkdocs.yml").read_text(encoding="utf-8") + assert "best-practices/python/typing-in-python.md:" in text + assert "tutorials/best-practices-integration/event-driven-microservices" in text