Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion agents/docs/commands-reference.md
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ help text).
| `fbuild show` | Show daemon logs or other introspection. | `fbuild help show` |
| `fbuild device` | List / inspect connected devices the daemon knows about. | `fbuild help device` |
| `fbuild purge` | Purge cached packages — full purge or LRU-only via `--gc`. | `fbuild help purge` |
| `fbuild lnk` | Manage `.lnk` resource pointers (fetch / verify / add). | `fbuild help lnk` |
| `fbuild lnk` | Manage `.fetch` blob pointers (fetch / verify / add). `.lnk` is still read for pointers written before FastLED/fbuild#1369; FastLED's runtime `.lnk` asset links are a different format and are skipped. | `fbuild help lnk` |

## Serial-port introspection (FastLED/fbuild#686)

Expand Down
146 changes: 142 additions & 4 deletions crates/fbuild-build-esp/src/esp32/orchestrator/embed_stage.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
//! Wrap `process_embed_files` with `.lnk` resolution + objcopy target selection.
//! Wrap `process_embed_files` with blob-pointer resolution + objcopy target
//! selection.

use std::path::{Path, PathBuf};

Expand All @@ -15,19 +16,38 @@ fn expand_embed_entries(
lnk_leases: &mut Vec<fbuild_packages::lnk::MaterializedLnk>,
) -> Result<Vec<String>> {
let mut out = Vec::with_capacity(entries.len());
// A materialized target is named after the pointer's blob, so two
// pointers with the same blob name land on one path — the second
// overwrites the first and both embed entries end up holding the second
// blob's bytes. Refuse instead: a wrong asset embedded in firmware is
// not something the user can see went wrong.
// Keyed by `normalize_for_key`, not by `PathBuf` equality. Windows and
// macOS are case-insensitive, so `logo.bin.fetch` and `LOGO.bin.lnk`
// produce lexically distinct targets that are the *same file* — which is
// exactly the collision this guard exists to catch, and the one a plain
// comparison lets through.
let mut claimed: std::collections::HashMap<String, String> = std::collections::HashMap::new();
for entry in entries {
let p = if Path::new(entry).is_absolute() {
PathBuf::from(entry)
} else {
project_dir.join(entry)
};
if fbuild_packages::lnk::has_lnk_extension(&p) {
if fbuild_packages::lnk::is_blob_pointer(&p) {
let cache = lnk_cache.ok_or_else(|| {
fbuild_core::FbuildError::PackageError(
"disk cache unavailable; cannot resolve .lnk entries".to_string(),
"disk cache unavailable; cannot resolve blob-pointer (.fetch/.lnk) entries"
.to_string(),
)
})?;
let materialized = fbuild_packages::lnk::materialize_lnk_entry(&p, lnk_dir, cache)?;
let claim_key = fbuild_core::path::normalize_for_key(&materialized.target_path);
if let Some(first) = claimed.insert(claim_key, entry.clone()) {
return Err(fbuild_core::FbuildError::PackageError(format!(
"embed entries `{first}` and `{entry}` both materialize to {} — blob pointers are named after the blob they point at, so two of them cannot share one. Rename one, or drop the stale pointer if this is a leftover `.lnk` beside its `.fetch` replacement (FastLED/fbuild#1369).",
Comment thread
zackees marked this conversation as resolved.
materialized.target_path.display()
)));
}
out.push(materialized.target_path.to_string_lossy().into_owned());
lnk_leases.push(materialized);
} else {
Expand All @@ -37,7 +57,7 @@ fn expand_embed_entries(
Ok(out)
}

/// Resolve `.lnk` entries in `embed_files`/`embed_txtfiles` against the disk
/// Resolve blob-pointer entries in `embed_files`/`embed_txtfiles` against the disk
/// cache, then convert each entry into a linkable ELF object. Returns the
/// list of object files to be appended to the sketch link set.
#[allow(clippy::too_many_arguments)]
Expand Down Expand Up @@ -152,4 +172,122 @@ mod tests {
"the cache lease must release when embed processing ends"
);
}

/// Two pointers whose blob names match materialize to one path, because
/// the target is derived from the file name alone. The second silently
/// replaced the first and both embed entries then pointed at the same
/// bytes.
///
/// Pre-existing for two `.lnk` in different directories; FastLED/fbuild
/// #1369 adds the case where `foo.bin.fetch` and `foo.bin.lnk` sit in the
/// *same* directory, which is exactly what a half-finished migration
/// looks like. Silence is the wrong answer either way.
#[test]
fn colliding_blob_names_are_refused_rather_than_silently_overwritten() {
let cache_root = tempfile::tempdir().unwrap();
let cache = fbuild_packages::DiskCache::open_at(cache_root.path()).unwrap();
let project = tempfile::tempdir().unwrap();

let write_pointer = |rel: &str, body: &[u8]| {
let sha = format!("{:x}", Sha256::digest(body));
let url = format!("https://localhost.invalid/{rel}");
let archive_dir = cache.archive_dir(Kind::LnkBlobs, &url, &sha);
std::fs::create_dir_all(&archive_dir).unwrap();
let blob_path = archive_dir.join("blob.bin");
std::fs::write(&blob_path, body).unwrap();
cache
.record_archive(
Kind::LnkBlobs,
&url,
&sha,
&blob_path.to_string_lossy(),
body.len() as i64,
&sha,
)
.unwrap();
let path = project.path().join(rel);
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
std::fs::write(
&path,
format!(r#"{{"v":1,"url":"{url}","sha256":"{sha}"}}"#),
)
.unwrap();
};
write_pointer("logo.bin.fetch", b"the fetch blob");
write_pointer("logo.bin.lnk", b"the legacy blob");

let mut leases = Vec::new();
let error = expand_embed_entries(
&["logo.bin.fetch".to_string(), "logo.bin.lnk".to_string()],
project.path(),
&project.path().join("build/lnk"),
Some(&cache),
&mut leases,
)
.expect_err("two pointers cannot share one materialized path");
let message = error.to_string();
assert!(message.contains("logo.bin"), "{message}");
assert!(
message.contains("logo.bin.fetch") && message.contains("logo.bin.lnk"),
"the error must name both pointers, or it is unactionable: {message}"
);
}

/// FastLED/fbuild#1369 review: on Windows and macOS the filesystem folds
/// case, so `logo.bin.fetch` and `LOGO.bin.lnk` materialize to one file
/// while comparing unequal as paths. Keying the guard lexically let
/// exactly the collision it was written to catch slip through — on the
/// platforms where it actually happens.
#[test]
fn blob_names_differing_only_by_case_collide_on_case_insensitive_hosts() {
if !fbuild_core::platform::host::is_windows() && !fbuild_core::platform::host::is_macos() {
return; // case-sensitive host: these really are two distinct files
}

let cache_root = tempfile::tempdir().unwrap();
let cache = fbuild_packages::DiskCache::open_at(cache_root.path()).unwrap();
let project = tempfile::tempdir().unwrap();

let write_pointer = |rel: &str, body: &[u8]| {
let sha = format!("{:x}", Sha256::digest(body));
let url = format!("https://localhost.invalid/{rel}");
let archive_dir = cache.archive_dir(Kind::LnkBlobs, &url, &sha);
std::fs::create_dir_all(&archive_dir).unwrap();
let blob_path = archive_dir.join("blob.bin");
std::fs::write(&blob_path, body).unwrap();
cache
.record_archive(
Kind::LnkBlobs,
&url,
&sha,
&blob_path.to_string_lossy(),
body.len() as i64,
&sha,
)
.unwrap();
let path = project.path().join(rel);
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
std::fs::write(
&path,
format!(r#"{{"v":1,"url":"{url}","sha256":"{sha}"}}"#),
)
.unwrap();
};
write_pointer("data/logo.bin.fetch", b"the fetch blob");
write_pointer("assets/LOGO.bin.lnk", b"the legacy blob");

let mut leases = Vec::new();
let error = expand_embed_entries(
&[
"data/logo.bin.fetch".to_string(),
"assets/LOGO.bin.lnk".to_string(),
],
project.path(),
&project.path().join("build/lnk"),
Some(&cache),
&mut leases,
)
.expect_err("case-folded names name one file on this host");
assert!(error.to_string().contains("materialize to"), "{error}");
}
}
26 changes: 17 additions & 9 deletions crates/fbuild-cli/src/cli/args.rs
Original file line number Diff line number Diff line change
Expand Up @@ -620,12 +620,17 @@ pub enum Commands {
#[arg(short = 'm', long)]
matcher: Option<String>,
},
/// Manage `.lnk` resource pointers (fetch / verify / add).
/// Manage `.fetch` blob pointers (fetch / verify / add).
///
/// `.lnk` files are tiny JSON manifests checked into source control
/// `.fetch` files are tiny JSON manifests checked into source control
/// that point at remote binary blobs (sha256-verified). At build time
/// fbuild downloads + caches them; this command lets you operate on
/// them outside of a build.
///
/// `.lnk` is still read for pointers written before the split
/// (FastLED/fbuild#1369). Note that FastLED's *runtime* asset links are
/// also `.lnk` but a different, plain-text format read on-device by
/// `fl::parse_lnk` — those are not fbuild's and are skipped.
Lnk {
#[command(subcommand)]
action: LnkAction,
Expand Down Expand Up @@ -771,26 +776,29 @@ pub enum Commands {
/// Subcommands for `fbuild lnk`.
#[derive(Subcommand)]
pub enum LnkAction {
/// Walk the current dir (or a project root) and fetch every `.lnk`
/// referenced blob into the disk cache. Cache hits are no-ops.
/// Walk the current dir (or a project root) and fetch every
/// blob-pointer-referenced blob into the disk cache. Cache hits are
/// no-ops.
Pull {
/// Project root to scan. Defaults to the current directory.
project_dir: Option<String>,
},
/// Verify every `.lnk` blob in the cache matches its sha256, without
/// touching the network. Reports mismatches; exits non-zero on any.
/// Verify every pointed-at blob in the cache matches its sha256,
/// without touching the network. Reports mismatches; exits non-zero on
/// any.
Check {
/// Project root to scan. Defaults to the current directory.
project_dir: Option<String>,
},
/// Download a URL once, compute its sha256, and write a new `.lnk`
/// Download a URL once, compute its sha256, and write a new `.fetch`
/// JSON pointing at it. Useful for adding new resources without
/// hand-editing JSON.
Add {
/// URL to download.
url: String,
/// Where to write the `.lnk` file. Defaults to the URL's basename
/// + `.lnk` in the current directory.
/// Where to write the pointer. Defaults to the URL's basename with
/// a `.fetch` suffix, in the current directory; an explicit path is
/// used exactly as given.
#[arg(short = 'o', long)]
output: Option<String>,
},
Expand Down
28 changes: 22 additions & 6 deletions crates/fbuild-cli/src/cli/lnk.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
//! `fbuild lnk` subcommands.
//!
//! - `pull` — scan + fetch every .lnk's blob into the disk cache
//! - `pull` — scan + fetch every blob pointer's blob into the disk cache
//! - `check` — verify every cached blob's sha256 (no network)
//! - `add` — fetch a URL once, hash it, write a new .lnk pointing at it
//! - `add` — fetch a URL once, hash it, write a new `.fetch` pointing at it

use crate::output;

Expand Down Expand Up @@ -36,7 +36,10 @@ pub async fn run_lnk(
let root = resolve_root(project_dir, top_level_project_dir);
let discovered = scan_for_lnk(&root)?;
if discovered.is_empty() {
output::result(format!("no .lnk files found under {}", root.display()));
output::result(format!(
"no blob pointers (.fetch/.lnk) found under {}",
root.display()
));
return Ok(());
}
let cache = open_cache()?;
Expand Down Expand Up @@ -73,7 +76,10 @@ pub async fn run_lnk(
let root = resolve_root(project_dir, top_level_project_dir);
let discovered = scan_for_lnk(&root)?;
if discovered.is_empty() {
output::result(format!("no .lnk files found under {}", root.display()));
output::result(format!(
"no blob pointers (.fetch/.lnk) found under {}",
root.display()
));
return Ok(());
}
let cache = open_cache()?;
Expand Down Expand Up @@ -150,9 +156,19 @@ pub async fn run_lnk(
// Determine output path before downloading so we fail early on a
// bad output spec.
let basename = url.rsplit('/').next().unwrap_or("blob");
// FastLED/fbuild#1369: newly written blob pointers get `.fetch`,
// which is unambiguously fbuild's. `.lnk` stays FastLED's runtime
// asset link — a different format with a different consumer, and
// conflating the two produced a silently wrong URL on-device
// (FastLED/FastLED#4012). An explicit `--output` is honored as
// typed: the user naming a file is not the user asking to be
// corrected.
let output_path = match output_arg {
Some(p) => PathBuf::from(p),
None => PathBuf::from(format!("{basename}.lnk")),
None => PathBuf::from(format!(
"{basename}.{}",
fbuild_packages::lnk::BLOB_POINTER_EXTENSION
)),
};
if let Some(parent) = output_path.parent() {
if !parent.as_os_str().is_empty() {
Expand Down Expand Up @@ -201,7 +217,7 @@ pub async fn run_lnk(
});
let pretty = serde_json::to_string_pretty(&json).map_err(|e| {
fbuild_core::FbuildError::PackageError(format!(
"failed to serialize .lnk JSON: {e}"
"failed to serialize blob-pointer JSON: {e}"
))
})?;
let mut f = std::fs::File::create(&output_path).map_err(|e| {
Expand Down
12 changes: 9 additions & 3 deletions crates/fbuild-packages/tests/lnk_e2e.rs
Original file line number Diff line number Diff line change
Expand Up @@ -116,13 +116,16 @@ async fn lnk_pipeline_e2e_fetches_verifies_and_materializes() {
spawn_test_server(vec![("asset.bin".to_string(), blob_bytes.clone())]).await;
let url = format!("http://127.0.0.1:{port}/asset.bin");

// Set up a project tree with one .lnk pointing at our test server.
// Set up a project tree with one blob pointer aimed at our test
// server. `.fetch` is what `fbuild lnk add` writes as of
// FastLED/fbuild#1369; the legacy `.lnk` spelling gets its own
// end-to-end run below.
let work = tempdir();
let src_root = work.path().join("src");
let build_dir = work.path().join("build/resources");
let cache_dir = work.path().join("cache");

let lnk_path = src_root.join("data/asset.bin.lnk");
let lnk_path = src_root.join("data/asset.bin.fetch");
std::fs::create_dir_all(lnk_path.parent().unwrap()).unwrap();
let lnk_json = format!(
r#"{{"v":1,"url":"{url}","sha256":"{blob_sha}","size":{}}}"#,
Expand All @@ -134,7 +137,7 @@ async fn lnk_pipeline_e2e_fetches_verifies_and_materializes() {

// Scan finds the lnk.
let discovered = scan_for_lnk(&src_root).unwrap();
assert_eq!(discovered.len(), 1, "scanner should find the one .lnk");
assert_eq!(discovered.len(), 1, "scanner should find the one pointer");
assert_eq!(discovered[0].lnk.sha256, blob_sha);

// Materialize fetches + verifies + writes into the build tree.
Expand Down Expand Up @@ -179,6 +182,9 @@ async fn lnk_pipeline_rejects_sha_mismatch() {
let build_dir = work.path().join("build");
let cache_dir = work.path().join("cache");

// Deliberately the legacy `.lnk` spelling: pointers written before
// FastLED/fbuild#1369 must keep resolving, and this run is what
// proves it rather than a comment claiming so.
let lnk_path = src_root.join("x.bin.lnk");
std::fs::create_dir_all(&src_root).unwrap();
std::fs::write(
Expand Down
Loading
Loading