CIP-003: cross-node shard placement

Cargo.toml

@@ -8,6 +8,7 @@ members = [
     "node/crates/c0mpute-core",
     "node/crates/c0mpute-net",
     "node/crates/c0mpute-store",
+    "node/crates/c0mpute-placement",
     "node/crates/c0mpute-gateway",
     "node/crates/c0mpute-verify",
     "node/crates/c0mpute-update",
@@ -65,6 +66,7 @@ rpassword = "7"
 
 c0mpute-proto = { path = "node/crates/c0mpute-proto" }
 c0mpute-store = { path = "node/crates/c0mpute-store" }
+c0mpute-placement = { path = "node/crates/c0mpute-placement" }
 c0mpute-net = { path = "node/crates/c0mpute-net" }
 c0mpute-gateway = { path = "node/crates/c0mpute-gateway" }
 c0mpute-verify = { path = "node/crates/c0mpute-verify" }

docs/prds/003-shard-placement-transport.md

@@ -1,15 +1,15 @@
 ---
 cip: 003
 title: "Cross-node shard placement and streaming transport"
-status: Draft
+status: In progress
 authors:
   - anthony@profullstack.com
 created: 2026-08-29
 updated: 2026-08-29
 implements: DIP-0012 (0012-storage-plugin.md) Phase 3
 depends-on: 002
 blocks: 005, 006
-implementation:
+implementation: PR #23 (c0mpute-placement crate; HTTP transport over the CIP-002 shard endpoints; `c0mpute storage peer`)
 estimate: "3–4 weeks"
 ---
 
@@ -57,6 +57,20 @@ first thing this CIP fixes.
 
 ## Design
 
+### What shipped first: HTTP, not libp2p
+
+The plan below rewrites the libp2p protocol before placement can work. That
+ordering turned out to be unnecessary. CIP-002 already ships shard `PUT`/`GET`/
+`HEAD` endpoints that verify what they are given, so placement was built
+against a `ShardTransport` trait with an HTTP implementation on top of those —
+and cross-node placement works today, on a real multi-node testnet, with no
+libp2p changes at all.
+
+The streaming libp2p protocol below is still worth doing (it removes an HTTP
+hop between peers that are already connected, and gives repair a batched
+`Have` probe). It is now a *second implementation of an existing trait* rather
+than a prerequisite, which makes it independently schedulable.
+
 ### Fix the transport first
 
 `request_response::cbor::Behaviour<FetchRequest, FetchResponse>` buffers an
@@ -93,11 +107,27 @@ Given a block needing `n` hosts, score each candidate peer:
 ```
 score = reputation                        # c0mpute-verify::reputation, >= 0.9 required
       * uptime_30d                        # >= 0.99 required (CIP-001)
-      * free_disk_factor                  # committed - used, normalised
-      * (1 / (1 + rtt_ms / 100))          # prefer near peers, weakly
+      * (0.9 + 0.1 / (1 + rtt_ms / 100))  # prefer near peers, weakly
 ```
 
-Then select greedily under **diversity constraints**, in priority order:
+Free disk is a hard filter rather than a score term — a peer either has room
+for the shard or it does not.
+
+**The latency weighting is deliberately narrower than this CIP first
+specified.** A bare `1 / (1 + rtt/100)` factor makes a 400 ms peer score 20%
+below a 1 ms one, which is enough for a fast flaky node to outrank a slow
+reliable one. CIP-001 is explicit that availability drives durability and
+latency does not, so the term is scaled into a band where it separates
+otherwise-equal peers but cannot overturn a reputation gap. A unit test pins
+this.
+
+Greedy selection under a per-domain cap is **optimal, not heuristic**: "at
+most `max_per_domain` from each domain" is a partition matroid, and greedy is
+optimal over a matroid. So a `DiversityUnsatisfiable` result means no other
+assignment would have worked either — no backtracking, and no better answer
+being missed.
+
+Select greedily under **diversity constraints**, in priority order:
 
 1. No two shards of the same block on the same peer. (Hard.)
 2. At most `floor(parity / 2)` shards per ASN — 2 of 14 for `standard`. (Hard.)

docs/prds/README.md

@@ -66,7 +66,7 @@ Delivering read/write network storage for c0mpute, implementing
 |-----|-------|-----------|--------|
 | [001](001-storage-program.md) | Storage program: durability model, tiers, and economics | — | In progress |
 | [002](002-storage-http-api.md) | Storage HTTP API on the gateway | 001 | In progress |
-| [003](003-shard-placement-transport.md) | Cross-node shard placement and streaming transport | 002 | Draft |
+| [003](003-shard-placement-transport.md) | Cross-node shard placement and streaming transport | 002 | In progress |
 | [004](004-metadata-durability.md) | Metadata durability: manifests, volumes, and the root pointer | 002 | Draft |
 | [005](005-repair-daemon.md) | Auto-repair daemon | 003, 004 | Draft |
 | [006](006-challenges-metering-payouts.md) | Storage challenges, metering, and provider payouts | 003, 004 | Draft |

node/crates/c0mpute-cli/Cargo.toml

@@ -16,12 +16,15 @@ c0mpute-proto = { workspace = true }
 c0mpute-update = { workspace = true }
 c0mpute-secure-chat = { workspace = true }
 c0mpute-store = { workspace = true }
+c0mpute-placement = { workspace = true }
+reqwest = { workspace = true }
 c0mpute-gateway = { workspace = true }
 axum = { workspace = true }
 anyhow = { workspace = true }
 tokio = { workspace = true }
 tracing = { workspace = true }
 clap = { workspace = true }
+serde = { workspace = true }
 serde_json = { workspace = true }
 toml = { workspace = true }
 rpassword = { workspace = true }

node/crates/c0mpute-cli/src/main.rs

@@ -19,6 +19,7 @@
 //! The plugin form mirrors the URL namespace: c0mpute.com/transcode,
 //! c0mpute.com/coinpay, c0mpute.com/infernet.
 
+mod peers;
 mod storage;
 
 use std::path::PathBuf;
@@ -356,6 +357,15 @@ fn maybe_self_update(cli: &Cli) {
 fn maybe_self_update(_cli: &Cli) {}
 
 fn main() -> Result<()> {
+    // Rust ignores SIGPIPE, so writing to a closed pipe returns EPIPE and the
+    // stdlib panics on it. For a CLI that prints lists that is a crash on
+    // `c0mpute storage ls | head`, which is ordinary shell usage. Restore the
+    // default disposition so the process exits quietly instead.
+    #[cfg(unix)]
+    unsafe {
+        libc::signal(libc::SIGPIPE, libc::SIG_DFL);
+    }
+
     let cli = Cli::parse();
 
     // Opportunistic self-update on any command (throttled), so c0mpute stays

node/crates/c0mpute-cli/src/peers.rs

@@ -0,0 +1,160 @@
+//! The node's storage-peer registry (CIP-003).
+//!
+//! Peers are read from `<storage-root>/peers.json`. Gossipsub capability ads
+//! will populate this automatically once the storage role advertises itself;
+//! until then an operator adds peers explicitly, which is also what makes a
+//! deliberate small testnet possible.
+
+use std::path::{Path, PathBuf};
+
+use anyhow::{Context, Result};
+use c0mpute_placement::{PeerCatalog, PeerInfo};
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Default, Serialize, Deserialize)]
+pub struct PeerFile {
+    #[serde(default)]
+    pub peers: Vec<PeerInfo>,
+}
+
+pub fn peers_path(storage_root: &Path) -> PathBuf {
+    storage_root.join("peers.json")
+}
+
+pub fn load(storage_root: &Path) -> Result<PeerCatalog> {
+    let path = peers_path(storage_root);
+    if !path.exists() {
+        return Ok(PeerCatalog::default());
+    }
+    let bytes = std::fs::read(&path).with_context(|| format!("read {}", path.display()))?;
+    let file: PeerFile =
+        serde_json::from_slice(&bytes).with_context(|| format!("parse {}", path.display()))?;
+    Ok(PeerCatalog::new(file.peers))
+}
+
+pub fn save(storage_root: &Path, catalog: &PeerCatalog) -> Result<()> {
+    let path = peers_path(storage_root);
+    if let Some(parent) = path.parent() {
+        std::fs::create_dir_all(parent)?;
+    }
+    let file = PeerFile {
+        peers: catalog.peers().to_vec(),
+    };
+    let json = serde_json::to_vec_pretty(&file)?;
+    // Write-then-rename: a half-written peer list would be parsed as a
+    // smaller network on the next read, and placement decisions follow from
+    // exactly that number.
+    let tmp = path.with_extension("json.tmp");
+    std::fs::write(&tmp, &json)?;
+    std::fs::rename(&tmp, &path)?;
+    Ok(())
+}
+
+/// Build a peer record, filling in what can be inferred.
+///
+/// A peer with no ASN and a DNS endpoint has an `Unknown` failure domain and
+/// is excluded from placement by default — deliberately, since CIP-001's
+/// durability figures assume independent hosts. `prefix_from_endpoint`
+/// recovers a weak-but-real domain when the endpoint is a literal IP.
+pub fn build(
+    peer_id: String,
+    endpoint: String,
+    asn: Option<u32>,
+    region: Option<String>,
+) -> PeerInfo {
+    let ip_prefix = PeerInfo::prefix_from_endpoint(&endpoint);
+    PeerInfo {
+        peer_id,
+        endpoint,
+        // Until CIP-006's challenges produce real numbers, a manually added
+        // peer is taken at its word. Recorded here rather than hidden so the
+        // assumption is visible when reputation starts being measured.
+        reputation: 1.0,
+        uptime_30d: 1.0,
+        free_bytes: u64::MAX,
+        rtt_ms: 50,
+        asn,
+        region,
+        ip_prefix,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn tmpdir() -> PathBuf {
+        let d = std::env::temp_dir().join(format!(
+            "c0mpute-peers-test-{}",
+            std::time::SystemTime::now()
+                .duration_since(std::time::UNIX_EPOCH)
+                .unwrap()
+                .as_nanos()
+        ));
+        std::fs::create_dir_all(&d).unwrap();
+        d
+    }
+
+    #[test]
+    fn missing_file_is_an_empty_catalog_not_an_error() {
+        let c = load(&tmpdir()).unwrap();
+        assert!(c.is_empty());
+    }
+
+    #[test]
+    fn round_trips_through_disk() {
+        let dir = tmpdir();
+        let mut c = PeerCatalog::default();
+        c.upsert(build(
+            "a".into(),
+            "http://10.0.0.1:7780".into(),
+            Some(7),
+            None,
+        ));
+        c.upsert(build(
+            "b".into(),
+            "http://10.1.0.1:7780".into(),
+            Some(8),
+            None,
+        ));
+        save(&dir, &c).unwrap();
+
+        let back = load(&dir).unwrap();
+        assert_eq!(back.len(), 2);
+        assert_eq!(back.get("a").unwrap().asn, Some(7));
+        assert_eq!(back.domain_count(), 2);
+    }
+
+    #[test]
+    fn infers_an_ip_prefix_when_the_asn_is_unknown() {
+        let p = build("a".into(), "http://203.0.113.9:7780".into(), None, None);
+        assert_eq!(p.ip_prefix.as_deref(), Some("203.0.113"));
+        assert_eq!(
+            p.domain(),
+            c0mpute_placement::FailureDomain::IpPrefix("203.0.113".into())
+        );
+    }
+
+    #[test]
+    fn a_dns_endpoint_without_an_asn_has_no_domain() {
+        let p = build(
+            "a".into(),
+            "http://node.example.com:7780".into(),
+            None,
+            None,
+        );
+        assert_eq!(p.ip_prefix, None);
+        assert_eq!(p.domain(), c0mpute_placement::FailureDomain::Unknown);
+    }
+
+    #[test]
+    fn an_explicit_asn_wins_over_the_inferred_prefix() {
+        let p = build(
+            "a".into(),
+            "http://203.0.113.9:7780".into(),
+            Some(64512),
+            None,
+        );
+        assert_eq!(p.domain(), c0mpute_placement::FailureDomain::Asn(64512));
+    }
+}