From 68b60455d1a4c58213ce9de53fd256b541d587f2 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Oct 2026 00:23:51 +0000 Subject: [PATCH 01/21] core: immersive artwork classifier (mirror, extend or card per edge) A pure function of a cover's pixels and the faces the platform found: Mirror where a reflection shows nothing that looks wrong flipped (no close-up faces, no text or logos near the edge, not too busy), Extend where the edge is one colour or its columns are, Card otherwise. Both edges (bottom for portrait, right for landscape), the colours to carry on, and light/dark controls with the scrim they need for 4.5:1. Tuned on 504 covers from a real library with scripts/artwork-eval.py and the artwork_layout example. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LDCDUaQsSzQpKRVfkiE1FK --- crates/hocket-core/examples/artwork_layout.rs | 29 + crates/hocket-core/src/api.rs | 108 ++++ crates/hocket-core/src/artwork/mod.rs | 594 ++++++++++++++++++ crates/hocket-core/src/artwork/tests.rs | 309 +++++++++ crates/hocket-core/src/lib.rs | 1 + scripts/artwork-eval.py | 172 +++++ 6 files changed, 1213 insertions(+) create mode 100644 crates/hocket-core/examples/artwork_layout.rs create mode 100644 crates/hocket-core/src/artwork/mod.rs create mode 100644 crates/hocket-core/src/artwork/tests.rs create mode 100755 scripts/artwork-eval.py diff --git a/crates/hocket-core/examples/artwork_layout.rs b/crates/hocket-core/examples/artwork_layout.rs new file mode 100644 index 0000000..343096b --- /dev/null +++ b/crates/hocket-core/examples/artwork_layout.rs @@ -0,0 +1,29 @@ +//! Runs the immersive-artwork classifier over raw covers, for tuning (see `scripts/artwork-eval.py`). +//! +//! Reads lines of ` ` on stdin and writes one +//! `ArtworkLayout` JSON per line to stdout. +//! +//! ```text +//! cargo run -p hocket-core --example artwork_layout --release < manifest.txt > layouts.jsonl +//! ``` +use std::io::{BufRead, Write}; + +fn main() -> anyhow::Result<()> { + let stdout = std::io::stdout(); + let mut out = stdout.lock(); + for line in std::io::stdin().lock().lines() { + let line = line?; + let mut parts = line.splitn(4, ' '); + let (Some(path), Some(w), Some(h)) = (parts.next(), parts.next(), parts.next()) else { + continue; + }; + let request = parts.next().unwrap_or(""); + let rgba = std::fs::read(path)?; + writeln!( + out, + "{}", + hocket_core::artwork::layout_json(&rgba, w.parse()?, h.parse()?, request)? + )?; + } + Ok(()) +} diff --git a/crates/hocket-core/src/api.rs b/crates/hocket-core/src/api.rs index 14678a5..348446b 100644 --- a/crates/hocket-core/src/api.rs +++ b/crates/hocket-core/src/api.rs @@ -764,6 +764,114 @@ pub struct AutoplaySettings { pub filter_id: Option, } +// --------------------------------------------------------------------------- +// Immersive artwork (how the full player shows a cover; `crate::artwork`) +// --------------------------------------------------------------------------- + +/// How the full player shows a cover along one of its edges. +#[typeshare] +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub enum ArtworkStyle { + /// Edge to edge, continued past the edge by a flipped copy that blurs and fades with distance. + Mirror, + /// Edge to edge, continued past the edge by the colours along it ([`ArtworkEdge::edge_colors`]). + Extend, + /// A rounded card on the artwork's blurred colours (the player as it was). + Card, +} + +/// The `display.immersiveArtwork` setting. +#[typeshare] +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default)] +#[serde(rename_all = "camelCase")] +pub enum ImmersiveArtwork { + /// Mirror or extend where the cover suits it, a card otherwise. + #[default] + Automatic, + /// Mirror or extend every square cover. + Always, + /// Always a card. + Never, +} + +/// A detected face, as fractions of the image's width and height (origin top left). +#[typeshare] +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Default)] +#[serde(rename_all = "camelCase")] +pub struct FaceRect { + pub x: f64, + pub y: f64, + pub w: f64, + pub h: f64, +} + +/// What the platform hands [`crate::artwork::layout`] beside the pixels. +#[typeshare] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)] +#[serde(rename_all = "camelCase")] +pub struct ArtworkLayoutRequest { + #[serde(default)] + pub faces: Vec, + #[serde(default)] + pub preference: ImmersiveArtwork, +} + +/// The measurements a decision was made from (for tuning and the debug grid). +#[typeshare] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)] +#[serde(rename_all = "camelCase")] +pub struct ArtworkMetrics { + /// Share of the edge strip within a small colour distance of its median (1 = one flat colour). + pub flat: f64, + /// Mean colour distance of the edge strip from its own column colours (0 = every column is one colour). + pub drift: f64, + /// How much the column colours change from one column to the next. + pub rough: f64, + /// Small high-contrast shapes (text, logos, stickers) in the zone a reflection shows sharply. + pub marks: u32, + /// Mean colour change between neighbouring pixels in the lower 40%. + pub busy: f64, + pub faces: u32, +} + +/// How to show the cover with the continuation along one edge (bottom for a portrait layout, right +/// for a landscape one). Colours are `0xRRGGBB`. +#[typeshare] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct ArtworkEdge { + pub style: ArtworkStyle, + /// The colours along the edge, sampled evenly (left to right for the bottom edge, top to bottom + /// for the right edge): what [`ArtworkStyle::Extend`] carries on. All one colour when the edge is flat. + pub edge_colors: Vec, + /// What the continuation fades into, far from the artwork. + pub base_color: u32, + /// Dark text and controls over the continuation (it is light); light ones otherwise. + pub light: bool, + /// The opacity of the scrim (black under light text, white under dark text) that the controls + /// need over the continuation to reach 4.5:1. + pub scrim: f64, + /// Why this style, e.g. `mirror`, `extend:flat`, `card:marks` (debugging and tuning). + pub reason: String, + pub metrics: ArtworkMetrics, +} + +/// The full player's artwork layout for one cover. +#[typeshare] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct ArtworkLayout { + /// Art on top, continued downwards (phones, narrow windows). + pub bottom: ArtworkEdge, + /// Art on the left, continued to the right (wide windows). + pub right: ArtworkEdge, + /// The top strip (under the status bar and the player's header) is light: dark icons there. + pub top_light: bool, + /// Text or a logo sits in the top strip: start the artwork below the status bar. + pub top_marks: bool, +} + // --------------------------------------------------------------------------- // Lyrics (renderable model adapted from OpenSubsonic v2 structured lyrics) // --------------------------------------------------------------------------- diff --git a/crates/hocket-core/src/artwork/mod.rs b/crates/hocket-core/src/artwork/mod.rs new file mode 100644 index 0000000..1ac0a95 --- /dev/null +++ b/crates/hocket-core/src/artwork/mod.rs @@ -0,0 +1,594 @@ +//! Immersive artwork: how the full player shows a cover (docs/design.md, "Immersive artwork"). +//! +//! [`layout`] is a pure function of a cover's pixels and the faces the platform found in it. The +//! platform decodes (it already does, for the artwork colours) and detects faces (Android's +//! `android.media.FaceDetector`); the decision lives here so both platforms make the same one and it +//! is tested once. Three styles, per edge: +//! +//! - [`ArtworkStyle::Mirror`] when a reflection would show nothing that looks wrong flipped: no +//! faces, no text or logos near the edge, not so busy that the controls drown in it; +//! - [`ArtworkStyle::Extend`] when the edge is one colour (a plain background with text on it) or +//! its columns are (a horizon, a gradient): its colours carry on, nothing is flipped; +//! - [`ArtworkStyle::Card`] otherwise. +//! +//! Everything is measured on the cover resampled to [`N`]×[`N`], in OKLab (so colour distances are +//! roughly perceptual); the thresholds were tuned on a real library's covers with the evaluation +//! example (`cargo run -p hocket-core --example artwork_layout`). + +use crate::api::{ + ArtworkEdge, ArtworkLayout, ArtworkLayoutRequest, ArtworkMetrics, ArtworkStyle, FaceRect, + ImmersiveArtwork, +}; + +#[cfg(test)] +mod tests; + +/// Side of the square a cover is resampled to before measuring. +pub const N: usize = 128; +/// The strip along the edge that Extend continues (skipping the outermost row: borders, JPEG fringes). +const EDGE_ROWS: usize = 5; +/// The zone a reflection shows sharply (~12%); deeper than this it is blurred past reading. +const REFLECT_ROWS: usize = 16; +/// The zone whose busyness matters (~40%): what sits behind the controls, flipped. +const BUSY_ROWS: usize = 51; +/// The strip under the status bar and the player's header (~10%). +const TOP_ROWS: usize = 13; +/// Colours closer than this (OKLab distance) count as the same flat colour. +const FLAT_TOL: f32 = 0.05; +/// An edge this flat is extended even where a mirror would do: there is nothing to reflect. +const FLAT_PLAIN: f64 = 0.97; +/// An edge at least this flat is extended (the marks on it are ignored). +const FLAT_MIN: f64 = 0.85; +/// Local lightness contrast (OKLab L against the median of its surroundings) that makes a pixel part of a mark. +const MARK_CONTRAST: f32 = 0.16; +/// A mark's surroundings are plain (a background, not more texture) below this mean neighbour distance. +const PLAIN_NOISE: f32 = 0.03; +/// The coarse (large lettering) pass counts from this many marks. +const COARSE_MIN: u32 = 4; +/// The outermost row this share one colour: extend it, whatever is above (the art's own edge +/// carries on exactly, and nothing is flipped). +const EDGE_ROW_FLAT: f64 = 0.95; +/// This many small marks in the reflection zone rule a mirror out. +const MARKS_MAX: u32 = 2; +/// Busier than this (mean neighbour distance) rules a mirror out. +const BUSY_MAX: f64 = 0.045; +/// Column extension needs columns this steady and this smooth from one to the next. +const DRIFT_MAX: f64 = 0.03; +const ROUGH_MAX: f64 = 0.02; +/// Faces narrower than this share of the width are ignored: a figure seen whole reflects like +/// one standing by water; it is a close-up face that looks wrong upside down. +const FACE_MIN: f64 = 0.1; +/// How many colours [`ArtworkEdge::edge_colors`] carries. +const EDGE_SAMPLES: usize = 16; +/// Contrast the controls need over the continuation (WCAG AA for text). +const TARGET: f64 = 4.5; +/// The dark foreground used over light continuations (Material's light on-surface). +const DARK_FG: [f32; 3] = [ + 0x1c as f32 / 255.0, + 0x1b as f32 / 255.0, + 0x1f as f32 / 255.0, +]; + +/// Layout for a cover given as tightly packed RGBA8 `width`×`height` pixels. +pub fn layout( + rgba: &[u8], + width: u32, + height: u32, + request: &ArtworkLayoutRequest, +) -> ArtworkLayout { + let (w, h) = (width as usize, height as usize); + if w == 0 || h == 0 || rgba.len() < w * h * 4 { + return card_layout("card:invalid"); + } + let img = Img::from_rgba(rgba, w, h); + let square = (w as f64 / h as f64 - 1.0).abs() <= 0.03; + let faces: Vec = request + .faces + .iter() + .copied() + .filter(|f| f.w >= FACE_MIN) + .collect(); + let transposed_faces: Vec = faces + .iter() + .map(|f| FaceRect { + x: f.y, + y: f.x, + w: f.h, + h: f.w, + }) + .collect(); + let top = img.median(0..TOP_ROWS); + ArtworkLayout { + bottom: edge(&img, &faces, square, request.preference), + right: edge( + &img.transposed(), + &transposed_faces, + square, + request.preference, + ), + top_light: luminance(top) > 0.4, + top_marks: img.marks(0..TOP_ROWS) >= MARKS_MAX, + } +} + +/// [`layout`] over JSON, for the FFI crates: the request in, the layout out. +pub fn layout_json( + rgba: &[u8], + width: u32, + height: u32, + request_json: &str, +) -> Result { + let request: ArtworkLayoutRequest = if request_json.trim().is_empty() { + ArtworkLayoutRequest::default() + } else { + serde_json::from_str(request_json)? + }; + serde_json::to_string(&layout(rgba, width, height, &request)) +} + +fn card_layout(reason: &str) -> ArtworkLayout { + let edge = ArtworkEdge { + style: ArtworkStyle::Card, + edge_colors: vec![0; EDGE_SAMPLES], + base_color: 0, + light: false, + scrim: 0.0, + reason: reason.into(), + metrics: ArtworkMetrics::default(), + }; + ArtworkLayout { + bottom: edge.clone(), + right: edge, + top_light: false, + top_marks: false, + } +} + +/// The decision for the bottom edge of `img` (the right edge is the bottom of the transposed image). +fn edge(img: &Img, faces: &[FaceRect], square: bool, preference: ImmersiveArtwork) -> ArtworkEdge { + let strip = N - 1 - EDGE_ROWS..N - 1; + // The outermost row alone (a few pixels of the original): one colour means a border, a + // backdrop or a plain band, and the art can simply carry on in it. + let last_median = img.median(N - 1..N); + let last_row_flat = img.share_within(N - 1..N, last_median, FLAT_TOL) >= EDGE_ROW_FLAT; + let median = if last_row_flat { + last_median + } else { + img.median(strip.clone()) + }; + let flat = img.share_within(strip.clone(), median, FLAT_TOL); + let columns = img.column_colours(strip.clone()); + let drift = img.drift(strip.clone(), &columns); + let smooth = smooth_columns(&columns, 3); + let rough = columns + .iter() + .zip(&smooth) + .map(|(a, b)| dist(*a, *b) as f64) + .sum::() + / N as f64; + let marks = img.marks(N - REFLECT_ROWS..N); + let busy = img.busyness(N - BUSY_ROWS..N); + let face_count = faces.len() as u32; + let metrics = ArtworkMetrics { + flat, + drift, + rough, + marks, + busy, + faces: face_count, + }; + + let mirror_ok = face_count == 0 && marks < MARKS_MAX && busy <= BUSY_MAX; + let columns_ok = drift <= DRIFT_MAX && rough <= ROUGH_MAX && img.marks(strip.clone()) == 0; + let (style, reason): (ArtworkStyle, &str) = match preference { + ImmersiveArtwork::Never => (ArtworkStyle::Card, "card:never"), + _ if !square => (ArtworkStyle::Card, "card:notSquare"), + _ if flat >= FLAT_PLAIN => (ArtworkStyle::Extend, "extend:flat"), + // A plain background with anything on it (a caption, a sticker): its colour, not a reflection. + _ if flat >= FLAT_MIN && marks > 0 => (ArtworkStyle::Extend, "extend:flat"), + _ if mirror_ok => (ArtworkStyle::Mirror, "mirror"), + // Not mirrorable, but the art ends in one colour (a border, a band, a backdrop): carry it on. + _ if last_row_flat => (ArtworkStyle::Extend, "extend:flat"), + _ if flat >= FLAT_MIN => (ArtworkStyle::Extend, "extend:flat"), + // Smearing a person's clothes or skin down the screen looks wrong: people only extend flat. + _ if columns_ok && face_count == 0 => (ArtworkStyle::Extend, "extend:columns"), + ImmersiveArtwork::Always => (ArtworkStyle::Extend, "extend:always"), + ImmersiveArtwork::Automatic if face_count > 0 => (ArtworkStyle::Card, "card:faces"), + ImmersiveArtwork::Automatic if marks >= MARKS_MAX => (ArtworkStyle::Card, "card:marks"), + ImmersiveArtwork::Automatic => (ArtworkStyle::Card, "card:busy"), + }; + + let flat_edge = reason == "extend:flat"; + // Extending a busy edge (the Always fallback) smears it: much smoother columns. + let edge_cols = if flat_edge { + vec![median; N] + } else if reason == "extend:always" { + smooth_columns(&columns, 16) + } else { + smooth + }; + let edge_colors: Vec<[f32; 3]> = (0..EDGE_SAMPLES) + .map(|i| { + let span = &edge_cols[i * N / EDGE_SAMPLES..(i + 1) * N / EDGE_SAMPLES]; + mean_lab(span) + }) + .collect(); + let (base, behind): ([f32; 3], Vec<[f32; 3]>) = match style { + // The reflection under the controls is the lower part, flipped and blurred: block averages. + ArtworkStyle::Mirror => ( + img.median(N - BUSY_ROWS..N), + img.blocks(N - BUSY_ROWS..N, 16), + ), + _ => ( + if flat_edge { + median + } else { + mean_lab(&edge_cols) + }, + edge_colors.clone(), + ), + }; + let behind_rgb: Vec<[f32; 3]> = behind.iter().map(|c| srgb_of_lab(*c)).collect(); + let (light, scrim) = if style == ArtworkStyle::Card { + (false, 0.0) + } else { + foreground(&behind_rgb) + }; + ArtworkEdge { + style, + edge_colors: edge_colors.iter().map(|c| pack(srgb_of_lab(*c))).collect(), + base_color: pack(srgb_of_lab(base)), + light, + scrim, + reason: reason.into(), + metrics, + } +} + +/// Light or dark controls, whichever needs the lighter scrim over `behind` (sRGB 0..1), and that scrim. +fn foreground(behind: &[[f32; 3]]) -> (bool, f64) { + let need = |fg: [f32; 3], scrim: f32| -> f64 { + let fg_l = luminance_srgb(fg); + (0..=100) + .map(|i| i as f32 / 100.0) + .find(|&a| { + behind.iter().all(|c| { + let over = [0, 1, 2].map(|k| c[k] * (1.0 - a) + scrim * a); + contrast(fg_l, luminance_srgb(over)) >= TARGET + }) + }) + .unwrap_or(1.0) as f64 + }; + let white = need([1.0; 3], 0.0); + let dark = need(DARK_FG, 1.0); + if dark < white { + (true, dark) + } else { + (false, white) + } +} + +fn contrast(a: f64, b: f64) -> f64 { + (a.max(b) + 0.05) / (a.min(b) + 0.05) +} + +// --------------------------------------------------------------------------- +// The resampled image +// --------------------------------------------------------------------------- + +/// A cover resampled to N×N, in OKLab, row-major. +struct Img { + lab: Vec<[f32; 3]>, +} + +impl Img { + /// Box-filtered (area average, in linear light) from RGBA8; transparency composites over black. + fn from_rgba(rgba: &[u8], w: usize, h: usize) -> Img { + let lut: Vec = (0..256).map(|v| srgb_to_linear(v as f32 / 255.0)).collect(); + let mut lab = Vec::with_capacity(N * N); + for dy in 0..N { + let y0 = dy * h / N; + let y1 = ((dy + 1) * h / N).max(y0 + 1).min(h); + for dx in 0..N { + let x0 = dx * w / N; + let x1 = ((dx + 1) * w / N).max(x0 + 1).min(w); + let mut acc = [0f32; 3]; + for y in y0..y1 { + for x in x0..x1 { + let p = &rgba[(y * w + x) * 4..(y * w + x) * 4 + 4]; + let a = p[3] as f32 / 255.0; + for k in 0..3 { + acc[k] += lut[p[k] as usize] * a; + } + } + } + let n = ((y1 - y0) * (x1 - x0)) as f32; + lab.push(oklab_of_linear(acc.map(|v| v / n))); + } + } + Img { lab } + } + + fn transposed(&self) -> Img { + let mut lab = vec![[0f32; 3]; N * N]; + for y in 0..N { + for x in 0..N { + lab[x * N + y] = self.lab[y * N + x]; + } + } + Img { lab } + } + + fn at(&self, x: usize, y: usize) -> [f32; 3] { + self.lab[y * N + x] + } + + fn rows(&self, rows: std::ops::Range) -> impl Iterator + '_ { + self.lab[rows.start * N..rows.end * N].iter().copied() + } + + fn median(&self, rows: std::ops::Range) -> [f32; 3] { + median_lab(self.rows(rows).collect()) + } + + fn share_within(&self, rows: std::ops::Range, c: [f32; 3], tol: f32) -> f64 { + let (n, near) = self.rows(rows).fold((0, 0), |(n, near), p| { + (n + 1, near + (dist(p, c) <= tol) as usize) + }); + near as f64 / n.max(1) as f64 + } + + fn column_colours(&self, rows: std::ops::Range) -> Vec<[f32; 3]> { + (0..N) + .map(|x| median_lab(rows.clone().map(|y| self.at(x, y)).collect())) + .collect() + } + + fn drift(&self, rows: std::ops::Range, columns: &[[f32; 3]]) -> f64 { + let n = rows.len() * N; + rows.flat_map(|y| (0..N).map(move |x| (x, y))) + .map(|(x, y)| dist(self.at(x, y), columns[x]) as f64) + .sum::() + / n.max(1) as f64 + } + + fn busyness(&self, rows: std::ops::Range) -> f64 { + let mut sum = 0.0; + let mut n = 0; + for y in rows.start..rows.end.min(N - 1) { + for x in 0..N - 1 { + let p = self.at(x, y); + sum += (dist(p, self.at(x + 1, y)) + dist(p, self.at(x, y + 1))) as f64 / 2.0; + n += 1; + } + } + sum / n.max(1) as f64 + } + + /// Mean colours of `size`×`size` blocks over `rows`. + fn blocks(&self, rows: std::ops::Range, size: usize) -> Vec<[f32; 3]> { + let mut out = Vec::new(); + let mut y = rows.start; + while y < rows.end { + let y1 = (y + size).min(rows.end); + for bx in (0..N).step_by(size) { + let cells: Vec<[f32; 3]> = (y..y1) + .flat_map(|yy| (bx..(bx + size).min(N)).map(move |xx| (xx, yy))) + .map(|(x, y)| self.at(x, y)) + .collect(); + out.push(mean_lab(&cells)); + } + y = y1; + } + out + } + + /// Small, high-contrast shapes (glyphs, logos, stickers) in `rows`: connected groups of pixels + /// whose lightness stands out from the median of their surroundings. A median keeps edges, + /// so the boundary between two large areas never stands out; thin strokes and small shapes do. + fn marks(&self, rows: std::ops::Range) -> u32 { + // Two scales: body text and stickers, then large lettering whose strokes are thicker + // than the small window (it would make the median itself). The coarse pass also picks up + // the odd bold shape in a photo, so it only counts when it finds a word's worth. + let coarse = self.marks_at(rows.clone(), 7, N / 5); + self.marks_at(rows, 3, N / 9) + .max(if coarse >= COARSE_MIN { coarse } else { 0 }) + } + + fn marks_at(&self, rows: std::ops::Range, r: usize, max_h: usize) -> u32 { + let mut window = Vec::with_capacity((2 * r + 1) * (2 * r + 1)); + let mut median = |x: usize, y: usize| { + window.clear(); + for yy in y.saturating_sub(r)..(y + r + 1).min(N) { + for xx in x.saturating_sub(r)..(x + r + 1).min(N) { + window.push(self.at(xx, yy)[0]); + } + } + let mid = window.len() / 2; + *window.select_nth_unstable_by(mid, f32::total_cmp).1 + }; + let (top, h) = (rows.start, rows.len()); + let mut on = vec![false; N * h]; + for y in rows.clone() { + for x in 0..N { + on[(y - top) * N + x] = (self.at(x, y)[0] - median(x, y)).abs() > MARK_CONTRAST; + } + } + let mut seen = vec![false; N * h]; + let mut count = 0; + let mut stack = Vec::new(); + for start in 0..N * h { + if !on[start] || seen[start] { + continue; + } + seen[start] = true; + stack.push(start); + let (mut x0, mut x1, mut y0, mut y1, mut area) = (N, 0, h, 0, 0); + while let Some(i) = stack.pop() { + let (x, y) = (i % N, i / N); + area += 1; + (x0, x1, y0, y1) = (x0.min(x), x1.max(x), y0.min(y), y1.max(y)); + for (dx, dy) in [ + (-1i32, -1i32), + (0, -1), + (1, -1), + (-1, 0), + (1, 0), + (-1, 1), + (0, 1), + (1, 1), + ] { + let (nx, ny) = (x as i32 + dx, y as i32 + dy); + if nx < 0 || ny < 0 || nx >= N as i32 || ny >= h as i32 { + continue; + } + let j = ny as usize * N + nx as usize; + if on[j] && !seen[j] { + seen[j] = true; + stack.push(j); + } + } + } + // A glyph or a small logo counts once; a run of glyphs that merged (bold text: the + // gaps between letters stand out from a median the letters made bright) counts as a + // few; anything taller than a line of text is a shape's detail, and a thin line right + // across the cover is a line, not text. + let (bw, bh) = (x1 - x0 + 1, y1 - y0 + 1); + // Marks touching the outer half of the zone count double: a reflection shows those + // sharpest (a Parental Advisory sticker in a bottom corner is enough on its own). + if area >= 2 && bh <= max_h && self.plain_around(&on, top, h, (x0, y0, x1, y1)) { + let weight = if top + y1 >= N - h / 2 && top > 0 { + 2 + } else { + 1 + }; + count += weight + * if bw <= N / 6 { + 1 + } else if bw <= N * 3 / 4 { + 3 + } else { + 0 + }; + } + } + count + } + + /// Whether what surrounds a mark (its box grown by 2, minus the mark's own pixels) is smooth: + /// text and logos sit on a clean background (a flat colour or a gentle gradient); the bright + /// details of foliage, gravel or a crowd sit among more of themselves, and a reflection of + /// those reads as texture, not as text. + fn plain_around( + &self, + on: &[bool], + top: usize, + h: usize, + (x0, y0, x1, y1): (usize, usize, usize, usize), + ) -> bool { + let (xa, xb, ya, yb) = ( + x0.saturating_sub(2), + (x1 + 3).min(N), + y0.saturating_sub(2), + (y1 + 3).min(h), + ); + let (mut sum, mut n) = (0f32, 0); + for y in ya..yb { + for x in xa..xb.saturating_sub(1) { + if !on[y * N + x] && !on[y * N + x + 1] { + sum += dist(self.at(x, y + top), self.at(x + 1, y + top)); + n += 1; + } + } + } + n > 0 && sum / n as f32 <= PLAIN_NOISE + } +} + +fn smooth_columns(columns: &[[f32; 3]], radius: usize) -> Vec<[f32; 3]> { + (0..columns.len()) + .map(|x| mean_lab(&columns[x.saturating_sub(radius)..(x + radius + 1).min(columns.len())])) + .collect() +} + +fn median_lab(mut cells: Vec<[f32; 3]>) -> [f32; 3] { + if cells.is_empty() { + return [0.0; 3]; + } + let mid = cells.len() / 2; + [0, 1, 2].map(|k| { + cells.sort_by(|a, b| a[k].total_cmp(&b[k])); + cells[mid][k] + }) +} + +fn mean_lab(cells: &[[f32; 3]]) -> [f32; 3] { + let n = cells.len().max(1) as f32; + [0, 1, 2].map(|k| cells.iter().map(|c| c[k]).sum::() / n) +} + +fn dist(a: [f32; 3], b: [f32; 3]) -> f32 { + ((a[0] - b[0]).powi(2) + (a[1] - b[1]).powi(2) + (a[2] - b[2]).powi(2)).sqrt() +} + +// --------------------------------------------------------------------------- +// Colour +// --------------------------------------------------------------------------- + +fn srgb_to_linear(c: f32) -> f32 { + if c <= 0.04045 { + c / 12.92 + } else { + ((c + 0.055) / 1.055).powf(2.4) + } +} + +fn linear_to_srgb(c: f32) -> f32 { + let c = c.clamp(0.0, 1.0); + if c <= 0.0031308 { + c * 12.92 + } else { + 1.055 * c.powf(1.0 / 2.4) - 0.055 + } +} + +fn oklab_of_linear([r, g, b]: [f32; 3]) -> [f32; 3] { + let l = (0.412_221_46 * r + 0.536_332_55 * g + 0.051_445_995 * b).cbrt(); + let m = (0.211_903_5 * r + 0.680_699_5 * g + 0.107_396_96 * b).cbrt(); + let s = (0.088_302_46 * r + 0.281_718_85 * g + 0.629_978_7 * b).cbrt(); + [ + 0.210_454_26 * l + 0.793_617_8 * m - 0.004_072_047 * s, + 1.977_998_5 * l - 2.428_592_2 * m + 0.450_593_7 * s, + 0.025_904_037 * l + 0.782_771_77 * m - 0.808_675_77 * s, + ] +} + +fn linear_of_oklab([l, a, b]: [f32; 3]) -> [f32; 3] { + let l_ = (l + 0.396_337_78 * a + 0.215_803_76 * b).powi(3); + let m_ = (l - 0.105_561_346 * a - 0.063_854_17 * b).powi(3); + let s_ = (l - 0.089_484_18 * a - 1.291_485_5 * b).powi(3); + [ + 4.076_741_7 * l_ - 3.307_711_6 * m_ + 0.230_969_94 * s_, + -1.268_438 * l_ + 2.609_757_4 * m_ - 0.341_319_38 * s_, + -0.004_196_086_3 * l_ - 0.703_418_6 * m_ + 1.707_614_7 * s_, + ] +} + +fn srgb_of_lab(c: [f32; 3]) -> [f32; 3] { + linear_of_oklab(c).map(linear_to_srgb) +} + +/// WCAG relative luminance of an OKLab colour. +fn luminance(c: [f32; 3]) -> f64 { + luminance_srgb(srgb_of_lab(c)) +} + +fn luminance_srgb(c: [f32; 3]) -> f64 { + let [r, g, b] = c.map(|v| srgb_to_linear(v.clamp(0.0, 1.0)) as f64); + 0.2126 * r + 0.7152 * g + 0.0722 * b +} + +fn pack(c: [f32; 3]) -> u32 { + let [r, g, b] = c.map(|v| (v.clamp(0.0, 1.0) * 255.0).round() as u32); + (r << 16) | (g << 8) | b +} diff --git a/crates/hocket-core/src/artwork/tests.rs b/crates/hocket-core/src/artwork/tests.rs new file mode 100644 index 0000000..ca49a65 --- /dev/null +++ b/crates/hocket-core/src/artwork/tests.rs @@ -0,0 +1,309 @@ +use super::*; + +const S: usize = 256; + +struct Canvas { + w: usize, + h: usize, + px: Vec, +} + +impl Canvas { + fn new(w: usize, h: usize, rgb: [u8; 3]) -> Canvas { + let mut px = Vec::with_capacity(w * h * 4); + for _ in 0..w * h { + px.extend_from_slice(&[rgb[0], rgb[1], rgb[2], 255]); + } + Canvas { w, h, px } + } + + fn set(&mut self, x: usize, y: usize, rgb: [u8; 3]) { + let i = (y * self.w + x) * 4; + self.px[i..i + 3].copy_from_slice(&rgb); + } + + fn fill(&mut self, x0: usize, y0: usize, w: usize, h: usize, rgb: [u8; 3]) { + for y in y0..(y0 + h).min(self.h) { + for x in x0..(x0 + w).min(self.w) { + self.set(x, y, rgb); + } + } + } + + /// A line of "glyphs": thin dark strokes, like a caption, with its top at `y`. + fn caption(&mut self, x: usize, y: usize, rgb: [u8; 3]) { + for i in 0..8 { + let gx = x + i * 14; + self.fill(gx, y, 3, 16, rgb); + self.fill(gx, y, 9, 3, rgb); + self.fill(gx + 6, y + 6, 3, 10, rgb); + } + } + + fn layout(&self, preference: ImmersiveArtwork, faces: Vec) -> ArtworkLayout { + layout( + &self.px, + self.w as u32, + self.h as u32, + &ArtworkLayoutRequest { faces, preference }, + ) + } + + fn auto(&self) -> ArtworkLayout { + self.layout(ImmersiveArtwork::Automatic, vec![]) + } +} + +/// A soft, low-contrast pattern: something a reflection continues nicely. +fn soft_texture() -> Canvas { + let mut c = Canvas::new(S, S, [0; 3]); + for y in 0..S { + for x in 0..S { + let v = ((x as f32 / 9.0).sin() * (y as f32 / 11.0).cos() * 0.5 + 0.5) * 120.0; + c.set( + x, + y, + [20 + v as u8, 40 + (v * 0.6) as u8, 50 + (v * 0.8) as u8], + ); + } + } + c +} + +/// Deterministic high-contrast noise: a busy cover. +fn noise() -> Canvas { + let mut c = Canvas::new(S, S, [0; 3]); + let mut seed = 0x2545_f491_u32; + for y in 0..S { + for x in 0..S { + seed ^= seed << 13; + seed ^= seed >> 17; + seed ^= seed << 5; + let [r, g, b, _] = seed.to_le_bytes(); + c.set(x, y, [r, g, b]); + } + } + c +} + +#[test] +fn a_soft_texture_is_mirrored() { + let l = soft_texture().auto(); + assert_eq!(l.bottom.style, ArtworkStyle::Mirror, "{:?}", l.bottom); + assert!(!l.bottom.light); +} + +#[test] +fn text_near_the_edge_of_a_plain_cover_extends_its_colour_instead_of_mirroring() { + // The "Do It" case: white, with a caption just above the bottom edge. + let mut c = Canvas::new(S, S, [255; 3]); + c.caption(120, S - 30, [10; 3]); + let l = c.auto(); + assert_eq!(l.bottom.style, ArtworkStyle::Extend, "{:?}", l.bottom); + assert_eq!(l.bottom.reason, "extend:flat"); + assert!( + l.bottom.edge_colors.iter().all(|&c| c == 0xffffff), + "{:x?}", + l.bottom.edge_colors + ); + // White stays white: dark controls, no scrim. + assert!(l.bottom.light); + assert!(l.bottom.scrim < 0.01, "{}", l.bottom.scrim); +} + +#[test] +fn text_in_the_reflection_zone_rules_a_mirror_out() { + let mut c = soft_texture(); + c.caption(40, S - 26, [255; 3]); + let l = c.auto(); + assert_ne!(l.bottom.style, ArtworkStyle::Mirror, "{:?}", l.bottom); + assert!( + l.bottom.metrics.marks >= MARKS_MAX, + "{:?}", + l.bottom.metrics + ); +} + +#[test] +fn a_busy_cover_is_a_card_unless_always() { + let c = noise(); + let l = c.auto(); + assert_eq!(l.bottom.style, ArtworkStyle::Card, "{:?}", l.bottom); + assert_eq!( + c.layout(ImmersiveArtwork::Always, vec![]).bottom.style, + ArtworkStyle::Extend + ); + assert_eq!( + c.layout(ImmersiveArtwork::Always, vec![]).bottom.reason, + "extend:always" + ); +} + +#[test] +fn never_is_always_a_card() { + let l = soft_texture().layout(ImmersiveArtwork::Never, vec![]); + assert_eq!(l.bottom.style, ArtworkStyle::Card); + assert_eq!(l.right.style, ArtworkStyle::Card); +} + +#[test] +fn a_face_rules_a_mirror_out_but_not_extending() { + let face = FaceRect { + x: 0.3, + y: 0.2, + w: 0.4, + h: 0.4, + }; + assert_ne!( + soft_texture() + .layout(ImmersiveArtwork::Automatic, vec![face]) + .bottom + .style, + ArtworkStyle::Mirror + ); + // A portrait on a plain backdrop still extends. + let mut c = Canvas::new(S, S, [200, 180, 160]); + c.fill(90, 40, 80, 120, [90, 60, 50]); + assert_eq!( + c.layout(ImmersiveArtwork::Automatic, vec![face]) + .bottom + .style, + ArtworkStyle::Extend + ); + // Tiny faces (a crowd far away) are ignored. + let tiny = FaceRect { + x: 0.5, + y: 0.5, + w: 0.06, + h: 0.06, + }; + assert_eq!( + soft_texture() + .layout(ImmersiveArtwork::Automatic, vec![tiny]) + .bottom + .style, + ArtworkStyle::Mirror + ); +} + +#[test] +fn columns_that_hold_their_colour_extend_column_by_column() { + // A left-to-right ramp, the same down every column, with a caption above the edge. + let mut c = Canvas::new(S, S, [0; 3]); + for y in 0..S { + for x in 0..S { + let v = (x * 255 / (S - 1)) as u8; + c.set(x, y, [v, v / 2, 255 - v]); + } + } + c.caption(60, S - 28, [255; 3]); + let l = c.auto(); + assert_eq!(l.bottom.style, ArtworkStyle::Extend, "{:?}", l.bottom); + assert_eq!(l.bottom.reason, "extend:columns"); + let first = l.bottom.edge_colors[0]; + let last = *l.bottom.edge_colors.last().unwrap(); + assert!( + first & 0xff > 0xc0 && last >> 16 > 0xc0, + "{first:06x} .. {last:06x}" + ); +} + +#[test] +fn the_right_edge_is_decided_on_its_own() { + // Busy on the left and bottom, a plain band down the right. + let mut c = noise(); + c.fill(S - 60, 0, 60, S, [30, 60, 120]); + let l = c.auto(); + assert_eq!(l.right.style, ArtworkStyle::Extend, "{:?}", l.right); + assert_eq!(l.bottom.style, ArtworkStyle::Card, "{:?}", l.bottom); + assert!(l + .right + .edge_colors + .iter() + .all(|&c| c == l.right.edge_colors[0])); +} + +#[test] +fn a_non_square_cover_is_a_card() { + let c = Canvas::new(S, S * 3 / 4, [255; 3]); + assert_eq!(c.auto().bottom.reason, "card:notSquare"); + assert_eq!( + c.layout(ImmersiveArtwork::Always, vec![]).bottom.style, + ArtworkStyle::Card + ); +} + +#[test] +fn a_dark_continuation_keeps_light_controls() { + let l = Canvas::new(S, S, [12, 14, 20]).auto(); + assert_eq!(l.bottom.style, ArtworkStyle::Extend); + assert!(!l.bottom.light); + assert!(l.bottom.scrim < 0.01); + assert!(!l.top_light); +} + +#[test] +fn a_mid_tone_continuation_gets_a_scrim() { + let l = Canvas::new(S, S, [125, 125, 125]).auto(); + assert!(l.bottom.scrim > 0.05, "{:?}", l.bottom); +} + +#[test] +fn text_across_the_top_is_flagged() { + let mut c = Canvas::new(S, S, [240, 235, 225]); + c.caption(30, 6, [20; 3]); + let l = c.auto(); + assert!(l.top_marks); + assert!(l.top_light); + assert!(!Canvas::new(S, S, [240, 235, 225]).auto().top_marks); +} + +#[test] +fn bad_input_is_a_card() { + assert_eq!( + layout(&[0; 10], 4, 4, &ArtworkLayoutRequest::default()) + .bottom + .style, + ArtworkStyle::Card + ); + assert_eq!( + layout(&[], 0, 0, &ArtworkLayoutRequest::default()) + .right + .style, + ArtworkStyle::Card + ); +} + +#[test] +fn small_covers_are_upsampled() { + let mut c = Canvas::new(40, 40, [255; 3]); + c.fill(0, 0, 40, 20, [0, 0, 0]); + assert_eq!(c.auto().bottom.style, ArtworkStyle::Extend); +} + +#[test] +fn json_round_trip() { + let c = soft_texture(); + let out = layout_json( + &c.px, + S as u32, + S as u32, + r#"{"faces":[],"preference":"always"}"#, + ) + .unwrap(); + let back: ArtworkLayout = serde_json::from_str(&out).unwrap(); + assert_eq!(back.bottom.style, ArtworkStyle::Mirror); + assert!(out.contains("\"edgeColors\"")); + assert!(layout_json(&c.px, S as u32, S as u32, "").is_ok()); + assert!(layout_json(&c.px, S as u32, S as u32, "{").is_err()); +} + +#[test] +fn colour_round_trip() { + for c in [[1.0, 0.0, 0.0], [0.2, 0.5, 0.9], [1.0, 1.0, 1.0]] { + let back = srgb_of_lab(oklab_of_linear(c.map(srgb_to_linear))); + for k in 0..3 { + assert!((back[k] - c[k]).abs() < 0.002, "{c:?} -> {back:?}"); + } + } +} diff --git a/crates/hocket-core/src/lib.rs b/crates/hocket-core/src/lib.rs index 4c86f6f..98396d6 100644 --- a/crates/hocket-core/src/lib.rs +++ b/crates/hocket-core/src/lib.rs @@ -14,6 +14,7 @@ pub mod core; pub mod util; pub mod actions; +pub mod artwork; pub mod audio; pub mod autoplay; pub mod cache; diff --git a/scripts/artwork-eval.py b/scripts/artwork-eval.py new file mode 100755 index 0000000..a40f318 --- /dev/null +++ b/scripts/artwork-eval.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 +"""Runs the immersive-artwork classifier (crates/hocket-core/src/artwork) over a folder of covers +and draws what the phone's full player would look like for each, for tuning by eye. + + pip install pillow numpy opencv-python-headless + cargo build -p hocket-core --example artwork_layout --release + scripts/artwork-eval.py + +Writes /layouts.jsonl (one per cover: file, faces, layout) and /sheet-NN.jpg contact +sheets grouped by the style chosen. Faces come from OpenCV's frontal Haar cascade, standing in for +Android's FaceDetector (also frontal-only). +""" +import json, os, subprocess, sys +import numpy as np +from PIL import Image, ImageDraw, ImageFilter, ImageFont + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +BIN = os.path.join(ROOT, "target", "release", "examples", "artwork_layout") +W, H = 240, 520 # a phone, in "dp" +DECODE = 256 # what the platform hands the core (Android decodes downsampled) + + +def faces_of(img): + """Faces as fractions of the image. With ARTWORK_EVAL_YUNET pointing at OpenCV Zoo's YuNet + model (face_detection_yunet_2023mar.onnx) a modern detector; otherwise the frontal Haar + cascade, roughly what Android's built-in FaceDetector finds.""" + import cv2 + yunet = os.environ.get("ARTWORK_EVAL_YUNET") + if yunet: + det = cv2.FaceDetectorYN.create(yunet, "", (320, 320), 0.7, 0.3, 50) + bgr = np.array(img.convert("RGB").resize((320, 320)))[:, :, ::-1].copy() + _, found = det.detect(bgr) + return [] if found is None else [{"x": float(f[0]) / 320, "y": float(f[1]) / 320, "w": float(f[2]) / 320, "h": float(f[3]) / 320} for f in found] + g = np.array(img.convert("L").resize((300, 300))) + cascade = cv2.CascadeClassifier(cv2.data.haarcascades + "haarcascade_frontalface_default.xml") + found = cascade.detectMultiScale(g, scaleFactor=1.1, minNeighbors=6, minSize=(18, 18)) + return [{"x": x / 300, "y": y / 300, "w": w / 300, "h": h / 300} for (x, y, w, h) in found] + + +def rgb(c): + return ((c >> 16) & 255, (c >> 8) & 255, c & 255) + + +def ramp(h, stops): + """A vertical alpha ramp (0..255) over h rows from (pos, alpha) stops.""" + ys = np.linspace(0, 1, h) + xs, vs = zip(*stops) + return np.interp(ys, xs, vs) + + +def mask(w, h, stops): + return Image.fromarray(np.repeat(ramp(h, stops)[:, None], w, 1).astype(np.uint8), "L") + + +def render(art, layout, edge="bottom"): + e = layout[edge] + style = e["style"] + base = rgb(e["baseColor"]) + phone = Image.new("RGB", (W, H), base) + if style == "card": + bg = art.resize((64, 64)).filter(ImageFilter.GaussianBlur(10)).resize((W, H)) + phone.paste(bg) + phone = Image.blend(phone, Image.new("RGB", (W, H), (0, 0, 0)), 0.5) + side = W - 32 + card = art.resize((side, side)) + m = Image.new("L", (side, side), 0) + ImageDraw.Draw(m).rounded_rectangle((0, 0, side - 1, side - 1), 16, fill=255) + phone.paste(card, (16, 56), m) + fg = (255, 255, 255) + else: + top = art.resize((W, W)) + below = H - W + # Far from the artwork every style ends on its blurred colours (the Card backdrop; moving + # on the device); a flat extension stays its one colour all the way down. + backdrop = art.resize((48, 48)).filter(ImageFilter.GaussianBlur(8)).resize((W, H)).crop((0, W, W, H)) + flat = e["reason"] == "extend:flat" + cont = Image.new("RGB", (W, below), base) if flat else backdrop + if style == "mirror": + flip = top.transpose(Image.FLIP_TOP_BOTTOM) + sharp = Image.new("RGB", (W, below), base); sharp.paste(flip) + mid = sharp.filter(ImageFilter.GaussianBlur(5)) + soft = sharp.filter(ImageFilter.GaussianBlur(16)) + cont.paste(soft, (0, 0), mask(W, below, [(0, 255), (0.45, 255), (0.85, 0)])) + cont.paste(mid, (0, 0), mask(W, below, [(0, 255), (0.08, 255), (0.25, 0)])) + cont.paste(sharp, (0, 0), mask(W, below, [(0, 255), (0.04, 0)])) + else: + # The edge's colours carried down, softening sideways with distance (sharp at the seam, + # a wide blur by mid-way), then fading into the backdrop. + cols = [rgb(c) for c in e["edgeColors"]] + row = Image.fromarray(np.array([cols], np.uint8)).resize((W, 1), Image.BILINEAR) + sharp = row.resize((W, below)) + wide = row.resize((W, 8)).filter(ImageFilter.GaussianBlur(28)).crop((0, 4, W, 5)).resize((W, below)) + soft = row.resize((W, 8)).filter(ImageFilter.GaussianBlur(10)).crop((0, 4, W, 5)).resize((W, below)) + cont.paste(wide, (0, 0), mask(W, below, [(0, 255), (0.5, 255), (0.9, 0)])) + cont.paste(soft, (0, 0), mask(W, below, [(0, 255), (0.12, 255), (0.4, 0)])) + cont.paste(sharp, (0, 0), mask(W, below, [(0, 255), (0.12, 0)])) + # Feather the seam over the artwork's last few rows. + f = 10 + top.paste(sharp.crop((0, 0, W, f)), (0, W - f), mask(W, f, [(0, 0), (1, 255)])) + phone.paste(top, (0, 0)) + phone.paste(cont, (0, W)) + light = e["light"] + fg = (28, 27, 31) if light else (255, 255, 255) + scrim = (255, 255, 255) if light else (0, 0, 0) + a = int(e["scrim"] * 255) + if a: + ov = Image.new("RGB", (W, H - W), scrim) + phone.paste(ov, (0, W), mask(W, H - W, [(0, 0), (0.25, a), (1, a)])) + # Status bar / header legibility over the top of the art. + if layout["topLight"] != light: + phone.paste(Image.new("RGB", (W, 48), scrim), (0, 0), mask(W, 48, [(0, 110), (1, 0)])) + d = ImageDraw.Draw(phone) + # Mock controls: title, artist, seek bar, transport, mode pills. + y = W + 50 if style != "card" else 56 + W - 32 + 18 + d.rounded_rectangle((16, y, 150, y + 14), 4, fill=fg) + d.rounded_rectangle((16, y + 22, 110, y + 32), 4, fill=tuple(int(v * 0.7 + (128 if fg[0] < 128 else 0) * 0.3) for v in fg)) + d.line((16, y + 56, W - 16, y + 56), fill=fg, width=2) + d.rounded_rectangle((16, y + 74, 66, y + 110), 14, outline=fg, width=2) + d.rounded_rectangle((74, y + 74, 166, y + 110), 14, fill=fg) + d.rounded_rectangle((174, y + 74, W - 16, y + 110), 14, outline=fg, width=2) + # Header text. + d.rounded_rectangle((16, 30, 100, 40), 3, fill=fg) + return phone + + +def main(src, out): + os.makedirs(out, exist_ok=True) + raw = os.path.join(out, "raw"); os.makedirs(raw, exist_ok=True) + names = sorted(f for f in os.listdir(src) if not f.startswith(".")) + arts, manifest = {}, [] + for f in names: + try: + img = Image.open(os.path.join(src, f)).convert("RGB") + except Exception: + continue + w, h = img.size + s = DECODE / max(w, h) + small = img.resize((max(1, round(w * s)), max(1, round(h * s))), Image.BOX) + p = os.path.join(raw, f + ".rgba") + open(p, "wb").write(small.convert("RGBA").tobytes()) + faces = faces_of(img) + arts[f] = (img, faces) + manifest.append(f"{p} {small.size[0]} {small.size[1]} " + json.dumps({"faces": faces, "preference": "automatic"})) + res = subprocess.run([BIN], input="\n".join(manifest) + "\n", capture_output=True, text=True, check=True) + layouts = [json.loads(l) for l in res.stdout.splitlines()] + rows = [{"file": f, "faces": arts[f][1], "layout": l} for f, l in zip(arts, layouts)] + with open(os.path.join(out, "layouts.jsonl"), "w") as fh: + for r in rows: + fh.write(json.dumps(r) + "\n") + order = {"mirror": 0, "extend": 1, "card": 2} + rows.sort(key=lambda r: (order[r["layout"]["bottom"]["style"]], r["layout"]["bottom"]["reason"], r["file"])) + font = ImageFont.load_default() + per, cols = 24, 8 + for n in range(0, len(rows), per): + chunk = rows[n:n + per] + sheet = Image.new("RGB", (cols * (W + 8) + 8, ((len(chunk) + cols - 1) // cols) * (H + 40) + 8), (40, 40, 40)) + d = ImageDraw.Draw(sheet) + for i, r in enumerate(chunk): + x, y = 8 + (i % cols) * (W + 8), 8 + (i // cols) * (H + 40) + sheet.paste(render(arts[r["file"]][0], r["layout"]), (x, y)) + b = r["layout"]["bottom"]; m = b["metrics"] + d.text((x, y + H + 2), f'{n + i}: {b["reason"]} faces {m["faces"]}', fill=(255, 255, 255), font=font) + d.text((x, y + H + 16), f'fl {m["flat"]:.2f} dr {m["drift"]:.3f} mk {m["marks"]} bz {m["busy"]:.3f}', fill=(200, 200, 200), font=font) + sheet.save(os.path.join(out, f"sheet-{n // per:02d}.jpg"), quality=82) + counts = {} + for r in rows: + k = r["layout"]["bottom"]["reason"]; counts[k] = counts.get(k, 0) + 1 + print(json.dumps(counts, indent=1)) + + +if __name__ == "__main__": + main(sys.argv[1], sys.argv[2]) From ad15761eb032a3ce5ab1840183fb2097585d0fe7 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Oct 2026 00:42:15 +0000 Subject: [PATCH 02/21] Immersive artwork: FFI, setting, and the desktop fullscreen player - artwork_layout over UniFFI (Android) and napi (desktop): RGBA8 pixels and an ArtworkLayoutRequest in, an ArtworkLayout out. - display.immersiveArtwork (automatic | always | never), device-local. - Desktop: a wide window puts the artwork at its full height on the left and carries it on to the right; a tall one at its full width on top, carried on below; anything in between keeps the card. The continuation is drawn once per cover and window size on a canvas over the fluid background (reflection or the edge's colours, softening with distance), with the scrim the core sized and dark controls over light covers. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LDCDUaQsSzQpKRVfkiE1FK --- crates/hocket-android/src/lib.rs | 17 ++ crates/hocket-core/src/settings/registry.rs | 9 + crates/hocket-core/src/settings/strings.rs | 1 + crates/hocket-node/src/lib.rs | 13 ++ desktop/src/main/core-host.ts | 2 + desktop/src/main/fake-core/index.ts | 1 + desktop/src/main/index.ts | 2 + desktop/src/main/ipc.ts | 22 ++- desktop/src/preload/index.ts | 4 +- desktop/src/renderer/lib/immersive.ts | 164 ++++++++++++++++++ desktop/src/renderer/styles/now-playing.css | 30 ++++ .../src/renderer/views/FullscreenPlayer.tsx | 56 +++++- desktop/src/renderer/views/Settings.tsx | 4 + desktop/src/shared/bridge-types.ts | 11 +- desktop/src/shared/settings-keys.ts | 2 + desktop/src/shared/strings.ts | 4 + 16 files changed, 335 insertions(+), 7 deletions(-) create mode 100644 desktop/src/renderer/lib/immersive.ts diff --git a/crates/hocket-android/src/lib.rs b/crates/hocket-android/src/lib.rs index a208daa..8d8794b 100644 --- a/crates/hocket-android/src/lib.rs +++ b/crates/hocket-android/src/lib.rs @@ -205,6 +205,23 @@ pub fn init_logging(level: String) { .try_init(); } +/// How the full player shows a cover (`hocket_core::artwork`): its pixels as tightly packed RGBA8 +/// plus an `ArtworkLayoutRequest` JSON (the faces found in it, the setting) in, an `ArtworkLayout` +/// JSON out. Pure and quick (the cover is resampled to 128 px); call it off the main thread. +#[uniffi::export] +pub fn artwork_layout( + rgba: Vec, + width: u32, + height: u32, + request_json: String, +) -> Result { + hocket_core::artwork::layout_json(&rgba, width, height, &request_json).map_err(|e| { + HocketError::Failed { + reason: e.to_string(), + } + }) +} + #[uniffi::export] pub fn core_version() -> String { env!("CARGO_PKG_VERSION").to_string() diff --git a/crates/hocket-core/src/settings/registry.rs b/crates/hocket-core/src/settings/registry.rs index d3a4f46..f2bfadb 100644 --- a/crates/hocket-core/src/settings/registry.rs +++ b/crates/hocket-core/src/settings/registry.rs @@ -75,6 +75,9 @@ pub mod keys { pub const DISPLAY_ACCENT: &str = "display.accent"; pub const DISPLAY_DYNAMIC_COLOUR: &str = "display.dynamicColour"; pub const DISPLAY_QUEUE_PANEL_SPLIT: &str = "display.queuePanelSplit"; + /// The full player's artwork: `automatic` (mirror or extend where the cover suits it, a card + /// otherwise), `always` (mirror or extend every square cover) or `never` (always a card). + pub const DISPLAY_IMMERSIVE_ARTWORK: &str = "display.immersiveArtwork"; pub const ACTIONS_ORDER_CONTEXT_MENU: &str = "actions.order.contextMenu"; pub const ACTIONS_ORDER_SIDEBAR: &str = "actions.order.sidebar"; pub const ACTIONS_ORDER_MEDIA_SESSION: &str = "actions.order.mediaSession"; @@ -389,6 +392,12 @@ pub static REGISTRY: &[SettingDef] = &[ }, || json!(0.5), ), + def( + DISPLAY_IMMERSIVE_ARTWORK, + Local, + SettingKind::Enum(&["automatic", "always", "never"]), + || json!("automatic"), + ), json_def( ACTIONS_ORDER_CONTEXT_MENU, Synced, diff --git a/crates/hocket-core/src/settings/strings.rs b/crates/hocket-core/src/settings/strings.rs index 1f21824..a4cb6a7 100644 --- a/crates/hocket-core/src/settings/strings.rs +++ b/crates/hocket-core/src/settings/strings.rs @@ -33,6 +33,7 @@ pub const TEXTS: &[SettingText] = &[ t("display.theme", "Theme", "Follow the system, or force light or dark."), t("display.accent", "Accent colour", "Accent colour as a hex value, or empty for the default."), t("display.dynamicColour", "Dynamic colour", "Derive the accent from the current artwork."), + t("display.immersiveArtwork", "Immersive artwork", "Run the artwork edge to edge, carried on below it by a reflection or by its own colours, except close-up photos of people and busy covers, which sit as a card on their own colours."), t("display.queuePanelSplit", "Queue / lyrics split", "Position of the divider between the queue and lyrics panels."), t("actions.order.contextMenu", "Context menu items", "Which actions appear in context menus, and in what order."), t("actions.order.sidebar", "Sidebar items", "Which sections appear in the sidebar, and in what order."), diff --git a/crates/hocket-node/src/lib.rs b/crates/hocket-node/src/lib.rs index b13fd3b..7b6a081 100644 --- a/crates/hocket-node/src/lib.rs +++ b/crates/hocket-node/src/lib.rs @@ -104,6 +104,19 @@ pub fn init_logging(level: String) { .try_init(); } +/// How the full player shows a cover (`hocket_core::artwork`): its pixels as tightly packed RGBA8 +/// plus an `ArtworkLayoutRequest` JSON in, an `ArtworkLayout` JSON out. Pure and quick. +#[napi(catch_unwind)] +pub fn artwork_layout( + rgba: Buffer, + width: u32, + height: u32, + request_json: String, +) -> Result { + hocket_core::artwork::layout_json(&rgba, width, height, &request_json) + .map_err(|e| Error::from_reason(e.to_string())) +} + /// Version of the addon crate (same as the workspace version). #[napi(catch_unwind)] pub fn core_version() -> String { diff --git a/desktop/src/main/core-host.ts b/desktop/src/main/core-host.ts index 6027e03..2e31df4 100644 --- a/desktop/src/main/core-host.ts +++ b/desktop/src/main/core-host.ts @@ -26,6 +26,8 @@ export interface NativeModule { initLogging(level: string): void; coreVersion(): string; apiSchemaVersion(): number; + /** The immersive-artwork classifier (addons built before it existed lack it). */ + artworkLayout?(rgba: Buffer, width: number, height: number, requestJson: string): string; mediaSessionAvailable(): boolean; } diff --git a/desktop/src/main/fake-core/index.ts b/desktop/src/main/fake-core/index.ts index 31c07e2..d67632e 100644 --- a/desktop/src/main/fake-core/index.ts +++ b/desktop/src/main/fake-core/index.ts @@ -1799,6 +1799,7 @@ export class FakeCore implements CoreHandle { ["display.accent", null, "deviceLocal"], ["display.dynamicColour", true, "deviceLocal"], ["display.queuePanelSplit", 0.5, "deviceLocal"], + ["display.immersiveArtwork", "automatic", "deviceLocal"], ["actions.order.contextMenu", [], "accountSynced"], ["actions.order.sidebar", [], "accountSynced"], ["actions.order.mediaSession", [], "accountSynced"], diff --git a/desktop/src/main/index.ts b/desktop/src/main/index.ts index e0eb552..e61cb44 100644 --- a/desktop/src/main/index.ts +++ b/desktop/src/main/index.ts @@ -125,11 +125,13 @@ async function main(): Promise { onOpenUri: (uri) => broadcastDeepLink(uri), }); + const classify = native?.module.artworkLayout?.bind(native.module); installIpc({ core, windows, artwork, credentials, + artworkLayout: core.kind === "native" ? classify : undefined, meta: () => ({ version: app.getVersion(), platform, diff --git a/desktop/src/main/ipc.ts b/desktop/src/main/ipc.ts index edede04..dc995ca 100644 --- a/desktop/src/main/ipc.ts +++ b/desktop/src/main/ipc.ts @@ -6,10 +6,10 @@ // writes a file (ExportNsp) may only target a path a save dialog just chose. import { BrowserWindow, clipboard, dialog, ipcMain, shell } from "electron"; import { readFile, writeFile } from "node:fs/promises"; -import type { Command, Query, QueryResult } from "@core/api"; +import type { ArtworkLayout, Command, Query, QueryResult } from "@core/api"; import type { CoreHandle } from "@shared/core-handle"; import type { AppMeta, AppPrefs, OpenDialogRequest, OpenedTextFile, SaveDialogRequest, WindowControl } from "@shared/bridge-types"; -import { IPC } from "@shared/bridge-types"; +import { ARTWORK_LAYOUT_MAX_SIDE, IPC } from "@shared/bridge-types"; import { t } from "@shared/strings"; import type { ArtworkRegistry } from "./artwork"; import { extractSecrets, parseConfigDocument, resolveSecretReferences } from "./config-secrets"; @@ -26,6 +26,13 @@ export interface IpcDeps { onNetworkReport: (online: boolean) => void; prefs: { get(): AppPrefs; set(patch: Partial): AppPrefs }; credentials: { passwordFor(url: string, username: string): string | undefined }; + /** The core's immersive-artwork classifier (JSON in and out), when the native core has it. */ + artworkLayout?: (rgba: Buffer, width: number, height: number, requestJson: string) => string; +} + +/** A side for `artworkLayout`: a whole number of pixels from 1 to ARTWORK_LAYOUT_MAX_SIDE. */ +function isSide(v: unknown): v is number { + return typeof v === "number" && Number.isInteger(v) && v >= 1 && v <= ARTWORK_LAYOUT_MAX_SIDE; } const MAX_TEXT_FILE = 16 * 1024 * 1024; @@ -107,6 +114,17 @@ export function installIpc(deps: IpcDeps): void { return result; }); + ipcMain.handle(IPC.artworkLayout, (e, rgba: unknown, width: unknown, height: unknown, request: unknown): ArtworkLayout | undefined => { + if (!isOurs(windows, e.sender) || !deps.artworkLayout) return undefined; + if (!(rgba instanceof Uint8Array) || !isSide(width) || !isSide(height) || rgba.length !== width * height * 4 || !isObject(request)) return undefined; + try { + return JSON.parse(deps.artworkLayout(Buffer.from(rgba.buffer, rgba.byteOffset, rgba.byteLength), width, height, JSON.stringify(request))) as ArtworkLayout; + } catch (err) { + console.error("[ipc] artworkLayout failed", err); + return undefined; + } + }); + ipcMain.handle(IPC.meta, (e): AppMeta | undefined => { if (!isOurs(windows, e.sender)) return undefined; const win = BrowserWindow.fromWebContents(e.sender); diff --git a/desktop/src/preload/index.ts b/desktop/src/preload/index.ts index d459063..a8ddefa 100644 --- a/desktop/src/preload/index.ts +++ b/desktop/src/preload/index.ts @@ -1,7 +1,7 @@ // Preload: installs the typed `window.hocket` bridge. contextIsolation is on, // sandbox is on, nothing from Node reaches the page. import { contextBridge, ipcRenderer } from "electron"; -import type { Command, Event, Query, QueryResult } from "@core/api"; +import type { ArtworkLayout, ArtworkLayoutRequest, Command, Event, Query, QueryResult } from "@core/api"; import type { AppMeta, AppPrefs, HocketBridge, OpenDialogRequest, OpenedTextFile, SaveDialogRequest, WindowControl, WindowState } from "@shared/bridge-types"; import { IPC } from "@shared/bridge-types"; @@ -61,6 +61,8 @@ const bridge: HocketBridge = { }, onDeepLink: (listener) => subscribe(IPC.deepLink, listener), onUiAction: (listener) => subscribe(IPC.uiAction, listener), + artworkLayout: (rgba: Uint8Array, width: number, height: number, request: ArtworkLayoutRequest) => + ipcRenderer.invoke(IPC.artworkLayout, rgba, width, height, request) as Promise, }; contextBridge.exposeInMainWorld("hocket", bridge); diff --git a/desktop/src/renderer/lib/immersive.ts b/desktop/src/renderer/lib/immersive.ts new file mode 100644 index 0000000..ec1316a --- /dev/null +++ b/desktop/src/renderer/lib/immersive.ts @@ -0,0 +1,164 @@ +// Immersive artwork (docs/design.md, "Immersive artwork"): the fullscreen +// player's artwork edge to edge, carried on past its edge by a reflection or by +// its own colours, or the usual card. The core decides which +// (hocket_core::artwork, over IPC); this side reads the cover's pixels from a +// canvas and draws the continuation, once per track and window size, as a +// still on a canvas: nothing redraws per frame. +import { useEffect, useState } from "react"; +import type { ArtworkEdge, ArtworkLayout, ImmersiveArtwork } from "@core/api"; +import { bridge } from "../core/bridge"; +import { useArtwork } from "../components/Artwork"; + +/** What the classifier gets: the cover at this side (it resamples to 128 itself). */ +const SIDE = 256; +const cache = new Map(); + +export function loadImage(url: string): Promise { + const img = new Image(); + img.crossOrigin = "anonymous"; + img.decoding = "async"; + img.src = url; + return img.decode().then(() => (img.width ? img : undefined), () => undefined); +} + +/** The layout for the cover at `url` (cached per URL and preference). */ +export async function classify(url: string, preference: ImmersiveArtwork): Promise { + const key = `${url}|${preference}`; + if (cache.has(key)) return cache.get(key); + const img = await loadImage(url); + let layout: ArtworkLayout | undefined; + if (img) { + const scale = Math.min(1, SIDE / Math.max(img.width, img.height)); + const w = Math.max(1, Math.round(img.width * scale)); + const h = Math.max(1, Math.round(img.height * scale)); + const c = document.createElement("canvas"); + c.width = w; + c.height = h; + const ctx = c.getContext("2d", { willReadFrequently: true }); + if (ctx) { + ctx.drawImage(img, 0, 0, w, h); + const data = ctx.getImageData(0, 0, w, h).data; + // No face detection on the desktop yet: faces only rule a mirror out, so a portrait may mirror. + layout = await bridge().artworkLayout(new Uint8Array(data.buffer, data.byteOffset, data.byteLength), w, h, { faces: [], preference }).catch(() => undefined); + } + } + cache.set(key, layout); + return layout; +} + +/** The layout for `coverArt`: undefined while it is worked out, without a cover, or without the native core. */ +export function useArtworkLayout(coverArt: string | undefined, preference: ImmersiveArtwork): ArtworkLayout | undefined { + const url = useArtwork(coverArt, 300); + const [layout, setLayout] = useState(undefined); + useEffect(() => { + setLayout(undefined); + if (!url) return; + let alive = true; + void classify(url, preference).then((l) => alive && setLayout(l)); + return () => { + alive = false; + }; + }, [url, preference]); + return layout; +} + +export type Orientation = "bottom" | "right"; + +const hex = (c: number) => `#${(c & 0xffffff).toString(16).padStart(6, "0")}`; + +/** `colors` averaged over a window of 2·radius + 1: the same edge, blurred sideways. */ +export function soften(colors: number[], radius: number): number[] { + return colors.map((_, i) => { + const span = colors.slice(Math.max(0, i - radius), i + radius + 1); + const ch = (shift: number) => Math.round(span.reduce((a, c) => a + ((c >> shift) & 255), 0) / span.length); + return (ch(16) << 16) | (ch(8) << 8) | ch(0); + }); +} + +/** + * Draws the continuation into `canvas` (sized to the player, in CSS px × dpr): for + * "bottom", everything below the artwork's square (the full width at the top); for + * "right", everything right of it (the full height at the left). Mirror: the artwork + * flipped, sharp at the seam and blurring with distance. Extend: the colours along + * the edge carried on, softening sideways with distance. Both fade out (to the + * fluid background beneath) except over a light continuation or one flat colour, + * which carry on in their own colour. Then the scrim the core worked out for 4.5:1. + */ +export function drawContinuation(canvas: HTMLCanvasElement, img: HTMLImageElement | undefined, edge: ArtworkEdge, orientation: Orientation, dpr: number): void { + const ctx = canvas.getContext("2d"); + if (!ctx) return; + ctx.setTransform(1, 0, 0, 1, 0, 0); + ctx.clearRect(0, 0, canvas.width, canvas.height); + // Work in "bottom" terms: for "right", transpose the canvas (x↔y) and the image. + const right = orientation === "right"; + const W = right ? canvas.height : canvas.width; + const H = right ? canvas.width : canvas.height; + const S = W; // the artwork's side: the full width (or, for "right", the full height) + if (H <= S) return; + let source: CanvasImageSource | undefined = img; + if (img && right) { + const t = document.createElement("canvas"); + t.width = img.width; + t.height = img.height; + const tc = t.getContext("2d"); + if (tc) { + tc.setTransform(0, 1, 1, 0, 0, 0); + tc.drawImage(img, 0, 0); + source = t; + } + } + if (right) ctx.setTransform(0, 1, 1, 0, 0, 0); + const region = H - S; + const flat = new Set(edge.edgeColors).size <= 1; + if (flat || edge.light) { + ctx.fillStyle = hex(edge.baseColor); + ctx.fillRect(0, S, W, region); + } + /** Draws `paint` into a band layer masked to `stops` ([fraction of S from the seam, alpha]). */ + const band = (stops: [number, number][], paint: (c: CanvasRenderingContext2D) => void) => { + const layer = document.createElement("canvas"); + layer.width = W; + layer.height = region; + const lc = layer.getContext("2d"); + if (!lc) return; + paint(lc); + lc.globalCompositeOperation = "destination-in"; + const g = lc.createLinearGradient(0, 0, 0, region); + for (const [y, a] of stops) g.addColorStop(Math.min(1, (y * S) / region), `rgba(0,0,0,${a})`); + lc.fillStyle = g; + lc.fillRect(0, 0, W, region); + ctx.drawImage(layer, 0, S); + }; + if (edge.style === "mirror" && source) { + const flipped = (blurPx: number) => (c: CanvasRenderingContext2D) => { + c.filter = blurPx > 0 ? `blur(${blurPx * dpr}px)` : "none"; + c.save(); + c.scale(1, -1); + c.drawImage(source, 0, -S, W, S); + c.restore(); + }; + band([[0, 1], [0.45, 1], [0.85, 0]], flipped(40)); + band([[0, 1], [0.08, 1], [0.25, 0]], flipped(12)); + band([[0, 1], [0.04, 0]], flipped(0)); + } else if (edge.style === "extend" && !flat) { + const across = (colors: number[]) => (c: CanvasRenderingContext2D) => { + const g = c.createLinearGradient(0, 0, W, 0); + colors.forEach((col, i) => g.addColorStop(colors.length > 1 ? i / (colors.length - 1) : 0, hex(col))); + c.fillStyle = g; + c.fillRect(0, 0, W, region); + }; + band([[0, 1], [0.5, 1], [0.9, 0]], across(soften(edge.edgeColors, 6))); + band([[0, 1], [0.12, 1], [0.4, 0]], across(soften(edge.edgeColors, 2))); + band([[0, 1], [0.12, 0]], across(edge.edgeColors)); + } + if (edge.scrim > 0) { + const rgb = edge.light ? "255,255,255" : "0,0,0"; + const g = ctx.createLinearGradient(0, S, 0, H); + g.addColorStop(0, `rgba(${rgb},0)`); + g.addColorStop(Math.min(1, (0.2 * S) / region), `rgba(${rgb},${edge.scrim})`); + g.addColorStop(1, `rgba(${rgb},${edge.scrim})`); + ctx.fillStyle = g; + ctx.fillRect(0, S, W, region); + } + ctx.setTransform(1, 0, 0, 1, 0, 0); +} diff --git a/desktop/src/renderer/styles/now-playing.css b/desktop/src/renderer/styles/now-playing.css index da3e072..ef3572d 100644 --- a/desktop/src/renderer/styles/now-playing.css +++ b/desktop/src/renderer/styles/now-playing.css @@ -230,3 +230,33 @@ /* While the player is open, snackbars drop in from the top instead of covering the transport. */ .app:has(.fullscreen.np) .toasts { bottom: auto; top: 16px; } + +/* ---- Immersive artwork (lib/immersive.ts) --------------------------------- + The artwork edge to edge: at the window's full height on the left ("right": + carried on to the right), or its full width on top ("bottom": carried on + below). The continuation is a still canvas over the fluid background; the + controls sit on it with the scrim the core sized. */ +.np-continuation { position: absolute; inset: 0; width: 100%; height: 100%; z-index: 0; pointer-events: none; animation: np-continuation-in var(--spring-effects-ms, 300ms) var(--spring-effects, var(--ease)); } +@keyframes np-continuation-in { from { opacity: 0; } } +.np[data-immersive] .np-fade { display: none; } +.np[data-immersive] .np-stage { position: static; } +.np[data-immersive] .np-main { position: absolute; left: 0; top: 0; z-index: 1; container-type: normal; } +.np[data-immersive] .np-art-slot { display: block; } +.np[data-immersive] .np-art { width: 100%; height: 100%; border-radius: 0; box-shadow: none; background: none; } +.np[data-immersive] .np-head, .np[data-immersive] .np-controls { position: relative; z-index: 2; } +.np[data-immersive="right"] .np-main { width: 100vh; height: 100vh; } +.np[data-immersive="right"] .np-stage, .np[data-immersive="right"][data-mode] .np-stage { + display: flex; flex-direction: column; width: auto; max-width: 560px; + margin: 0 clamp(24px, 4vw, 72px) 0 calc(100vh + clamp(32px, 4vw, 72px)); padding: 28px 0 36px; +} +.np[data-immersive="right"] .np-controls { margin-block: auto; gap: 18px; } +.np[data-immersive="bottom"] .np-main { width: 100vw; height: 100vw; } +.np[data-immersive="bottom"] .np-stage { padding-top: calc(100vw + 16px); height: auto; min-height: 100%; } +.np[data-immersive="bottom"] .np-head { position: absolute; top: 18px; left: 24px; right: 24px; } +/* Header legibility over the artwork's top when it disagrees with the controls. */ +.np[data-immersive="bottom"]:not([data-light])[data-top-light] .np-main::after { content: ""; position: absolute; inset: 0 0 auto; height: 22%; background: linear-gradient(rgba(0, 0, 0, 0.45), transparent); pointer-events: none; } +/* Over a light continuation (a white cover carried on in white): dark text and controls. */ +.np[data-light] { color: #1c1b1f; --fg: #1c1b1f; --fg-muted: rgba(28, 27, 31, 0.8); --fg-faint: rgba(28, 27, 31, 0.8); --bg-hover: rgba(0, 0, 0, 0.06); --bg-active: rgba(0, 0, 0, 0.1); --border: rgba(0, 0, 0, 0.15); --focus-ring: #1c1b1f; --accent-text: #1c1b1f; --accent-soft: rgba(0, 0, 0, 0.08); --selection: rgba(0, 0, 0, 0.12); } +.np[data-light] .np-play { background: #1c1b1f; color: #fff; } +.np[data-light] .m3-slider { --active: #1c1b1f; --handle: #1c1b1f; --stop: #1c1b1f; --inactive: rgba(0, 0, 0, 0.24); } +@media (prefers-reduced-motion: reduce) { .np-continuation { animation: none; } } diff --git a/desktop/src/renderer/views/FullscreenPlayer.tsx b/desktop/src/renderer/views/FullscreenPlayer.tsx index cce6a5c..f82813a 100644 --- a/desktop/src/renderer/views/FullscreenPlayer.tsx +++ b/desktop/src/renderer/views/FullscreenPlayer.tsx @@ -20,7 +20,7 @@ import { useEffect, useLayoutEffect, useRef, useState, type CSSProperties } from "react"; import { t } from "@shared/strings"; import type { Track } from "@core/api"; -import { useApp, type NowPlayingMode } from "../store/app"; +import { useApp, useSetting, type NowPlayingMode } from "../store/app"; import { useQuery } from "../store/queries"; import { executeAction } from "../store/actions"; import { bridge } from "../core/bridge"; @@ -36,7 +36,11 @@ import { openMenuFromButton } from "../components/ContextMenu"; import { fmtBytes, fmtDate, fmtTime } from "../lib/format"; import { trapTab, useReturnFocus } from "../lib/focus"; import { usePresence, type Presence } from "../lib/presence"; -import { usePrefersReducedMotion } from "../lib/media"; +import { useMediaQuery, usePrefersReducedMotion } from "../lib/media"; +import { drawContinuation, loadImage, useArtworkLayout, type Orientation } from "../lib/immersive"; +import { useArtwork } from "../components/Artwork"; +import { SK } from "@shared/settings-keys"; +import type { ArtworkEdge, ImmersiveArtwork } from "@core/api"; import { volumeValueText } from "../lib/a11y"; import { SPRING_EFFECTS, SPRING_FAST, SPRING_SPATIAL } from "../lib/spring"; @@ -185,10 +189,21 @@ function Player({ closing, motion, exitProps }: PlayerProps) { }; const repeatLabel = queue.repeat === "off" ? t("player.repeatOff") : queue.repeat === "all" ? t("player.repeatAll") : t("player.repeatOne"); const art = track ? :
; + // Immersive artwork: a wide window puts the artwork at its full height on the left, + // carried on to the right; a tall one at its full width on top, carried on below. + // In between (and in the other modes) the artwork is a card. + const preference = useSetting(SK.displayImmersiveArtwork, "automatic"); + const layout = useArtworkLayout(track?.coverArt, preference); + const wide = useMediaQuery(WIDE_IMMERSIVE); + const tall = useMediaQuery(TALL_IMMERSIVE); + const orientation: Orientation | undefined = mode === "art" && layout ? (wide ? "right" : tall ? "bottom" : undefined) : undefined; + const edge = orientation ? layout?.[orientation] : undefined; + const immersive = edge && edge.style !== "card" ? { edge, orientation: orientation as Orientation } : undefined; return ( -
+
+ {immersive && track ? : null}