diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0a6947e..d305c6e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -14,7 +14,13 @@ jobs: - uses: actions/checkout@v7 with: fetch-depth: 0 + submodules: recursive filter: blob:none + - uses: jdx/aube-action@v1 + with: + version: '2.5.1' + node-version: '26.5.0' + run-install: false - uses: pkgxdev/setup@v5 - run: sudo apt-get update && sudo apt-get install -y libasound2-dev libfontconfig1-dev libwayland-dev libxkbcommon-dev libxkbcommon-x11-dev libxcb1-dev libx11-xcb-dev libx11-dev libegl1-mesa-dev libvulkan-dev libssl-dev - uses: moonrepo/setup-toolchain@v0 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 30f3378..0288df8 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -15,6 +15,12 @@ jobs: - uses: actions/checkout@v7 with: fetch-depth: 0 + submodules: recursive + - uses: jdx/aube-action@v1 + with: + version: '2.5.1' + node-version: '26.5.0' + run-install: false - uses: pkgxdev/setup@v5 - uses: moonrepo/setup-toolchain@v0 with: @@ -103,6 +109,12 @@ jobs: - uses: actions/checkout@v7 with: fetch-depth: 0 + submodules: recursive + - uses: jdx/aube-action@v1 + with: + version: '2.5.1' + node-version: '26.5.0' + run-install: false - name: Install host-native LLVM env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -150,6 +162,8 @@ jobs: Expand-Archive (Join-Path $tools $archive) $tools (Get-ChildItem $tools -Recurse -Filter moon.exe | Select-Object -First 1).DirectoryName >> $env:GITHUB_PATH - run: rustup toolchain install 1.98.1 --profile minimal --component clippy,rustfmt --target ${{ matrix.target }} + - name: Bundle pinned YouTube.js source + run: moon exec --ignore-ci-checks youtubejs:bundle - name: Resolve desktop release version id: version shell: pwsh diff --git a/.gitignore b/.gitignore index 242c325..c45cd55 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,8 @@ /dist/ /.moon/cache/ /node_modules/ +/vendor/bundle/ +/vendor/node_modules/ /crates/*/dist/ diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..6bc488d --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "vendor/youtubejs"] + path = vendor/youtubejs + url = https://github.com/listenbox/YouTube.js.git diff --git a/.moon/workspace.yml b/.moon/workspace.yml index 49abd4f..754c351 100644 --- a/.moon/workspace.yml +++ b/.moon/workspace.yml @@ -1,5 +1,6 @@ defaultProject: client projects: + youtubejs: vendor client: . cli: crates/cli client-engine: crates/sync-engine diff --git a/Cargo.lock b/Cargo.lock index ab9ad10..efdcb02 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1310,6 +1310,33 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "cookie" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a373e3602691c3cdea496d2f0ee5935151e6168fe87739483c463db1b2f2f87" +dependencies = [ + "percent-encoding", + "time", + "version_check", +] + +[[package]] +name = "cookie_store" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b2c103cf610ec6cae3da84a766285b42fd16aad564758459e6ecf128c75206" +dependencies = [ + "cookie", + "document-features", + "idna", + "log", + "serde", + "serde_derive", + "time", + "url", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -4470,6 +4497,8 @@ version = "0.1.0" dependencies = [ "anyhow", "chrono", + "cookie", + "cookie_store", "ffmpeg-the-third", "futures-util", "hex", @@ -4483,6 +4512,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", + "sha1 0.10.7", "sha2 0.10.9", "tempfile", "tokio", @@ -7017,7 +7047,6 @@ checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ "base64", "bytes", - "futures-channel", "futures-core", "futures-util", "http", @@ -8359,17 +8388,6 @@ dependencies = [ "objc", ] -[[package]] -name = "tar" -version = "0.4.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" -dependencies = [ - "filetime", - "libc", - "xattr", -] - [[package]] name = "tauri-winrt-notification" version = "0.7.3" @@ -10375,16 +10393,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "xattr" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" -dependencies = [ - "libc", - "rustix", -] - [[package]] name = "xcb" version = "1.7.1" @@ -10512,20 +10520,15 @@ dependencies = [ name = "youtubei" version = "0.1.0" dependencies = [ - "flate2", "futures-util", "llrt_modules", "llrt_utils_watts", - "reqwest", "rquickjs", "serde", "serde_json", "sha1 0.10.7", - "sha2 0.10.9", "static_assertions", - "tar", "tokio", - "toml 0.8.23", "uuid", ] diff --git a/README.md b/README.md index ffa1aef..fa53071 100644 --- a/README.md +++ b/README.md @@ -11,13 +11,13 @@ A Rust monorepo for the Listenbox desktop app, CLI and shared synchronization en ## Build -Install Rust via rustup, [Moon](https://moonrepo.dev), [pkgx](https://pkgx.sh), [kache 0.27.0](https://github.com/kunobi-ninja/kache/releases/tag/v0.27.0), a C compiler, make and tar. Rust is pinned in `rust-toolchain.toml`. No JavaScript runtime or package manager is needed in this repository. +Install Rust via rustup, [Moon](https://moonrepo.dev), [pkgx](https://pkgx.sh), [kache 0.27.0](https://github.com/kunobi-ninja/kache/releases/tag/v0.27.0), a C compiler, make and tar. Rust is pinned in `rust-toolchain.toml`. Source builds also use Node.js 26 and [Aube](https://aube.sh); shipped applications need neither. ```sh # Install once per machine, outside a checkout (or use the prebuilt release). cargo install --locked kache --version 0.27.0 -git clone https://github.com/listenbox/client.git +git clone --recurse-submodules https://github.com/listenbox/client.git cd client moon run client:build moon ci @@ -25,7 +25,7 @@ moon ci The debug app is `crates/desktop/dist/listenbox-desktop`; the terminal executable is `crates/cli/dist/listenbox`. `moon run desktop:build-release` builds the production desktop executable for the host architecture at `crates/desktop/dist/release/listenbox-desktop`. `moon run desktop:dmg` packages its Apple Silicon build as a DMG. `moon run client:package` packages release CLI and desktop binaries with notices. -FFmpeg 9.0.2 is built from verified source by `tools/native-ffmpeg.rs`, linked with `ffmpeg-the-third`, and never run as a subprocess. The physical `youtubei` crate embeds a verified upstream bundle in QuickJS. Both applications are self-contained. See `THIRD-PARTY-NOTICES.txt` and the packaged FFmpeg source/license notice. +FFmpeg 9.0.2 is built from verified source by `tools/native-ffmpeg.rs`, linked with `ffmpeg-the-third`, and never run as a subprocess. The `youtubei` crate embeds our pinned `vendor/youtubejs` source build in QuickJS. Moon installs locked JavaScript build dependencies and creates the bundle before Cargo; Cargo never downloads a prebuilt YouTube.js bundle. Both applications are self-contained. See `THIRD-PARTY-NOTICES.txt` and the packaged FFmpeg source/license notice. ### Moon and Cargo @@ -204,6 +204,39 @@ listenbox shows order --show field-notes --episode ep_0123456789abcdef --episode Supplied episodes move to the front in sequence; other episodes retain their relative order. Sync restores playlist order for imported podcasts. +## YouTube sign-in checks + +When YouTube asks you to sign in, open **Settings → YouTube** in the desktop app +(⌘, on macOS or Ctrl+, on Windows). Paste a **Netscape cookies.txt** export and +choose **Save cookies**, then return to the podcast and choose **Sync now**. +The CLI accepts the same format, without requiring a Listenbox login to save it: + +```sh +listenbox youtube-cookies import /path/to/cookies.txt +listenbox shows sync youtube --show field-notes +listenbox youtube-cookies remove +``` + +Follow [the private-session export guide](https://listenbox.app/guides/import-youtube-as-a-podcast/#when-youtube-asks-you-to-sign-in). +Cookies stay in the shared profile's `youtube-cookies/jar.json`. The app never +shows the saved contents. A new import replaces the session; removal returns +future requests to anonymous access. Only YouTube receives these credentials; +media hosts and the Listenbox API do not. + +The sync engine reads immutable snapshots without a reader lock. Writers take +an OS file lock, compare the request's generation and per-cookie revisions, +merge unrelated updates, then sync and atomically replace the snapshot. Late +responses cannot overwrite a newer rotation, resurrect a deleted cookie, or +undo a user replacement/removal. Interrupted writes leave the prior complete +snapshot readable. Response cookies are committed before consuming the body. +Unix directories are private (0700) and snapshots are 0600; Windows uses the +user profile's inherited access controls. + +Tests use synthetic cookie exports and local YouTube fixtures. The optional +`verify_youtube` example is a manual live probe: it copies an explicitly supplied +export into a temporary profile, checks playlist extraction and media ranges, +and deletes that profile on exit. It never publishes and is not run by CI. + ## Interrupted work Both interfaces use the engine's persistent sync path. It opens `~/.config/listenbox/sync.sqlite` only for sync work and keeps media under `~/.config/listenbox/transfers`. Refinery applies embedded SQL migrations with strict history validation. Neither interface accesses SQLite directly. diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index e3774fc..769cd70 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -33,6 +33,11 @@ enum Command { slug: Option, source_url: String, }, + /// Import or remove the YouTube session shared with Listenbox desktop + YoutubeCookies { + #[command(subcommand)] + command: CookieCommand, + }, /// Manage shows by slug Shows { #[command(subcommand)] @@ -50,6 +55,14 @@ enum Command { }, } +#[derive(Subcommand)] +enum CookieCommand { + /// Replace saved cookies with a Netscape cookies.txt export + Import { file: std::path::PathBuf }, + /// Remove the saved session and return to anonymous access + Remove, +} + #[derive(Subcommand)] enum AuthCommand { Status, @@ -272,6 +285,16 @@ async fn run(cli: Cli, cancel: CancellationToken) -> Result<()> { let config = config::Config::load(cli.config.as_deref())?; let mut api = api::Api::new(config, cancel)?; match cli.command { + Command::YoutubeCookies { command } => { + let jar = listenbox_sync_engine::cookies::CookieJar::new(&api.config.directory); + tokio::task::spawn_blocking(move || match command { + CookieCommand::Import { file } => jar.import_file(&file), + CookieCommand::Remove => jar.remove(), + }) + .await??; + println!("YouTube cookies updated. Sync your podcast to continue."); + Ok(()) + } Command::Login => auth::login(&mut api).await, Command::Auth { command: AuthCommand::Logout, @@ -301,7 +324,8 @@ async fn run(cli: Cli, cancel: CancellationToken) -> Result<()> { Command::Shows { command } => commands::shows(&api, command).await, Command::Episodes { command } => episodes::run(&api, command).await, Command::Members { command } => commands::members(&api, command).await, - Command::Login + Command::YoutubeCookies { .. } + | Command::Login | Command::Auth { command: AuthCommand::Logout, } => unreachable!(), diff --git a/crates/desktop/examples/preview.rs b/crates/desktop/examples/preview.rs index 961060d..8334b18 100644 --- a/crates/desktop/examples/preview.rs +++ b/crates/desktop/examples/preview.rs @@ -21,6 +21,14 @@ mod workspace { view.catalog.shows[0].image_url = Some("https://artwork.example.test/missing.png".into()); } + pub fn show_settings(view: &mut Workspace, saved: bool) { + view.settings_open = true; + view.cookie_saved = saved; + } + pub fn show_cookie_recovery(view: &mut Workspace) { + view.error = Some(ErrorNotice::youtube_sign_in()); + } + pub fn show_import(view: &mut Workspace) { view.import_open = true; } @@ -177,6 +185,9 @@ fn main() -> anyhow::Result<()> { ("payment-dark", ThemeMode::Dark, 840., 600., true, false), ("quota-light", ThemeMode::Light, 1080., 760., true, false), ("quota-dark", ThemeMode::Dark, 840., 600., true, false), + ("settings-light", ThemeMode::Light, 1080., 760., true, false), + ("settings-dark", ThemeMode::Dark, 840., 600., true, false), + ("cookie-recovery", ThemeMode::Light, 840., 600., true, false), ("quit-light", ThemeMode::Light, 1080., 760., false, true), ("quit-dark", ThemeMode::Dark, 840., 600., true, true), ] { @@ -211,6 +222,12 @@ fn main() -> anyhow::Result<()> { if name == "workspace-dark" { workspace::show_selected_team(&mut view); } + if name.starts_with("settings-") { + workspace::show_settings(&mut view, name == "settings-dark"); + } + if name == "cookie-recovery" { + workspace::show_cookie_recovery(&mut view); + } if quitting { workspace::show_quit_notice(&mut view, cx); } diff --git a/crates/desktop/moon.yml b/crates/desktop/moon.yml index ef8d7e8..e20e711 100644 --- a/crates/desktop/moon.yml +++ b/crates/desktop/moon.yml @@ -109,7 +109,7 @@ tasks: runInCI: false build-release-windows-x64: description: Build and package a Windows x64 release, also runnable through Windows 11 ARM emulation. - deps: [client:codegen, client:native-ffmpeg-windows-x64] + deps: [client:codegen, youtubejs:bundle, client:native-ffmpeg-windows-x64] command: powershell.exe -NoProfile -ExecutionPolicy Bypass -File ../../tools/build-windows.ps1 -Architecture x64 -Stage Release inputs: ['@group(sources)', 'project://client?group=sources', 'project://client?group=notices'] outputs: [dist/windows-x64/**/*, dist/listenbox-desktop-windows-x64.exe, dist/listenbox-desktop-windows-x64.zip] @@ -119,7 +119,7 @@ tasks: cache: false build-release-windows-arm64: description: Build and package a native Windows ARM64 release on Windows ARM64. - deps: [client:codegen, client:native-ffmpeg-windows-arm64] + deps: [client:codegen, youtubejs:bundle, client:native-ffmpeg-windows-arm64] command: powershell.exe -NoProfile -ExecutionPolicy Bypass -File ../../tools/build-windows.ps1 -Architecture arm64 -Stage Release inputs: ['@group(sources)', 'project://client?group=sources', 'project://client?group=notices'] outputs: [dist/windows-arm64/**/*, dist/listenbox-desktop-windows-arm64.exe, dist/listenbox-desktop-windows-arm64.zip] diff --git a/crates/desktop/src/platform.rs b/crates/desktop/src/platform.rs index 71dee81..de0636b 100644 --- a/crates/desktop/src/platform.rs +++ b/crates/desktop/src/platform.rs @@ -5,7 +5,7 @@ use gpui_kit::{App, Entity, Menu, MenuItem, Window, actions}; #[path = "platform/windows.rs"] mod windows; -actions!(listenbox, [Logout, Quit]); +actions!(listenbox, [Logout, Quit, OpenSettings]); /// Query the actual shortcut key while its quit attempt is active. macOS can /// consume Command-key releases before they reach GPUI's focused view. @@ -48,7 +48,18 @@ pub fn install_actions(view: &Entity, cx: &mut App) { }); let quit = view.clone(); cx.on_action(move |_: &Quit, cx| quit.update(cx, |view, cx| view.shutdown(Shutdown::Quit, cx))); + cx.bind_keys([gpui_kit::KeyBinding::new( + if cfg!(target_os = "macos") { + "cmd-," + } else { + "ctrl-," + }, + OpenSettings, + None, + )]); cx.set_menus(vec![Menu::new("Listenbox").items([ + MenuItem::action("Settings…", OpenSettings), + MenuItem::separator(), MenuItem::action("Log out", Logout), MenuItem::separator(), MenuItem::action("Quit Listenbox", Quit), diff --git a/crates/desktop/src/workspace.rs b/crates/desktop/src/workspace.rs index 9fcf839..29b07d4 100644 --- a/crates/desktop/src/workspace.rs +++ b/crates/desktop/src/workspace.rs @@ -3,7 +3,7 @@ use crate::tokens::{self, Tokens}; use gpui_kit::component::{ Disableable, Icon, Sizable, button::{Button, ButtonVariants}, - input::{Input, InputState}, + input::{Input, InputState, TextareaState}, progress::Progress, spinner::Spinner, }; @@ -22,6 +22,8 @@ use tokio_util::{sync::CancellationToken, task::TaskTracker}; #[path = "workspace/episodes.rs"] mod episodes; +#[path = "workspace/settings.rs"] +mod settings; pub struct Workspace { episodes: Vec, @@ -56,6 +58,12 @@ pub struct Workspace { team_picker: bool, selected: Option, source: Entity, + settings_open: bool, + cookie_input: Entity, + cookie_busy: bool, + cookie_saved: bool, + cookie_message: Option, + cookie_error: Option, jobs: HashMap, reports: HashMap, error: Option, @@ -66,6 +74,7 @@ pub struct Workspace { struct ErrorNotice { message: String, upgrade_url: Option, + youtube_sign_in: bool, } impl From for ErrorNotice { @@ -73,11 +82,16 @@ impl From for ErrorNotice { Self { message, upgrade_url: None, + youtube_sign_in: false, } } } impl ErrorNotice { + fn youtube_sign_in() -> Self { + Self { message: "YouTube is asking you to sign in. Add fresh cookies in Settings, then use Sync now to continue this podcast.".into(), upgrade_url: None, youtube_sign_in: true } + } + fn import( error: anyhow::Error, client: &Client, @@ -85,7 +99,9 @@ impl ErrorNotice { created: bool, kind: &ShowSourceKind, ) -> Self { - if let Some(payment) = error.downcast_ref::() { + if error.is::() { + Self::youtube_sign_in() + } else if let Some(payment) = error.downcast_ref::() { let reason = if payment.0.is_empty() { "Payment is required to continue." } else { @@ -100,6 +116,7 @@ impl ErrorNotice { "Could not create podcast." } ), + youtube_sign_in: false, upgrade_url: team .and_then(|team| client.upgrade_url(team).ok()) .map(|url| { @@ -132,6 +149,8 @@ enum Message { Report(String, Report), Finished(String, anyhow::Result<()>), SyncDue, + CookieStatus(anyhow::Result), + CookiesSaved(bool, anyhow::Result<()>), Episodes( u64, bool, @@ -213,6 +232,12 @@ impl Workspace { team_picker: false, selected: None, source, + settings_open: false, + cookie_input: settings::cookie_input(window, cx), + cookie_busy: false, + cookie_saved: false, + cookie_message: None, + cookie_error: None, jobs: HashMap::new(), reports: HashMap::new(), error: None, @@ -354,6 +379,16 @@ impl Workspace { return; } match message { + Message::CookieStatus(result) => { + self.cookie_busy = false; + match result { + Ok(saved) => self.cookie_saved = saved, + Err(error) => self.cookie_error = Some(error.to_string()), + } + } + Message::CookiesSaved(saved, result) => { + self.cookies_received(saved, result, window, cx) + } Message::Episodes(request, append, result) => { self.episodes_received(request, append, result) } @@ -528,7 +563,15 @@ impl Workspace { self.reports .insert(slug, "Stopped. Progress is saved for the next sync.".into()); } else if let Err(error) = result { - self.reports.insert(slug, format!("Sync failed. {error:#}")); + if error.is::() { + self.error = Some(ErrorNotice::youtube_sign_in()); + self.reports.insert( + slug, + "YouTube sign-in required. Open Settings to add fresh cookies.".into(), + ); + } else { + self.reports.insert(slug, format!("Sync failed. {error:#}")); + } } } } @@ -537,6 +580,8 @@ impl Workspace { fn select(&mut self, slug: Option, _window: &mut Window, cx: &mut Context) { self.selected = slug; + self.settings_open = false; + self.cookie_input = settings::cookie_input(_window, cx); self.import_open = false; self.load_episodes(false, cx); self.error = None; @@ -815,7 +860,9 @@ impl Workspace { .child(Icon::new(assets::IconName::Plus).small()), ) .disabled(!self.loaded || self.importing || self.stopping.is_some()) - .on_click(cx.listener(|view, _, _, cx| { + .on_click(cx.listener(|view, _, window, cx| { + view.settings_open = false; + view.cookie_input = settings::cookie_input(window, cx); view.import_open = true; view.error = None; cx.notify(); @@ -1158,6 +1205,11 @@ impl Render for Workspace { div() .image_cache(self.artwork_cache.clone()) .track_focus(&self.focus) + .on_action( + cx.listener(|view, _: &crate::platform::OpenSettings, window, cx| { + view.open_settings(window, cx) + }), + ) .capture_key_down(cx.listener(|view, event: &KeyDownEvent, _, cx| { if event.keystroke.modifiers.platform && event.keystroke.key == "q" { cx.stop_propagation(); @@ -1208,6 +1260,15 @@ impl Render for Workspace { .when(self.loading || self.authenticating, |row| { row.child(Spinner::new().small()) }) + .child( + Button::new("open-settings") + .ghost() + .label("Settings") + .disabled(self.stopping.is_some()) + .on_click(cx.listener(|view, _, window, cx| { + view.open_settings(window, cx) + })), + ) .child( Button::new("reload") .ghost() @@ -1250,28 +1311,49 @@ impl Render for Workspace { .min_h_0() .overflow_y_scroll() .p(px(tokens::SPACE)) - .when_some(self.error.clone(), |pane, error| { - pane.child( - div() - .mb_4() - .flex() - .flex_col() - .items_start() - .gap_2() - .child(div().text_color(t.danger).child(error.message)) - .when_some(error.upgrade_url, |notice, url| { - notice.child( - Button::new("upgrade-plan") - .ghost() - .icon(assets::IconName::ExternalLink) - .label("Upgrade plan") - .on_click(move |_, _, cx| cx.open_url(&url)), - ) - }), - ) + .when_some( + self.error.clone().filter(|_| !self.settings_open), + |pane, error| { + pane.child( + div() + .mb_4() + .flex() + .flex_col() + .items_start() + .gap_2() + .child(div().text_color(t.danger).child(error.message)) + .when(error.youtube_sign_in, |notice| { + notice.child( + Button::new("youtube-sign-in-settings") + .primary() + .label("Open YouTube settings") + .on_click(cx.listener( + |view, _, window, cx| { + view.open_settings(window, cx) + }, + )), + ) + }) + .when_some(error.upgrade_url, |notice, url| { + notice.child( + Button::new("upgrade-plan") + .ghost() + .icon(assets::IconName::ExternalLink) + .label("Upgrade plan") + .on_click(move |_, _, cx| { + cx.open_url(&url) + }), + ) + }), + ) + }, + ) + .child(if self.settings_open { + self.settings(cx) + } else { + self.detail(window, cx) }) - .child(self.detail(window, cx)) - .when(self.loaded, |pane| { + .when(self.loaded && !self.settings_open, |pane| { pane.child(self.episode_list(cx)).child(self.transfers(cx)) }), ), diff --git a/crates/desktop/src/workspace/settings.rs b/crates/desktop/src/workspace/settings.rs new file mode 100644 index 0000000..6d1c073 --- /dev/null +++ b/crates/desktop/src/workspace/settings.rs @@ -0,0 +1,124 @@ +use super::*; +use gpui_kit::component::input::Textarea; +use listenbox_sync_engine::cookies; + +pub(super) fn cookie_input( + window: &mut Window, + cx: &mut Context, +) -> Entity { + cx.new(|cx| { + TextareaState::new(window, cx) + .rows(6) + .placeholder("Paste the complete Netscape cookies.txt export here") + }) +} +impl Workspace { + pub(super) fn open_settings(&mut self, window: &mut Window, cx: &mut Context) { + if self.settings_open || self.stopping.is_some() { + return; + } + self.settings_open = true; + self.cookie_message = None; + self.cookie_error = None; + self.cookie_input = cookie_input(window, cx); + if !self.cookie_busy { + self.cookie_busy = true; + let (jar, sender) = (self.client.cookie_jar(), self.sender.clone()); + self.tasks.spawn_on( + async move { + let result = tokio::task::spawn_blocking(move || jar.is_enabled()) + .await + .map_err(anyhow::Error::from) + .and_then(|result| result); + let _ = sender.send(Message::CookieStatus(result)); + }, + self.runtime.handle(), + ); + } + cx.notify(); + } + fn save_cookies(&mut self, remove: bool, cx: &mut Context) { + if self.cookie_busy || self.stopping.is_some() { + return; + } + let text = self.cookie_input.read(cx).value().to_string(); + self.cookie_busy = true; + self.cookie_error = None; + self.cookie_message = None; + self.cookie_input + .update(cx, |input, cx| input.set_disabled(true, cx)); + let (jar, sender) = (self.client.cookie_jar(), self.sender.clone()); + self.tasks.spawn_on( + async move { + let result = tokio::task::spawn_blocking(move || { + if remove { + jar.remove() + } else { + jar.import(&text) + } + }) + .await + .map_err(anyhow::Error::from) + .and_then(|result| result); + let _ = sender.send(Message::CookiesSaved(!remove, result)); + }, + self.runtime.handle(), + ); + cx.notify(); + } + pub(super) fn cookies_received( + &mut self, + saved: bool, + result: anyhow::Result<()>, + window: &mut Window, + cx: &mut Context, + ) { + self.cookie_busy = false; + self.cookie_input + .update(cx, |input, cx| input.set_disabled(false, cx)); + match result { + Ok(()) => { + self.cookie_saved = saved; + // Drop the editing history too: Undo must not reveal saved secrets. + self.cookie_input = cookie_input(window, cx); + self.cookie_message = Some( + if saved { + "Cookies saved. Return to your podcast and choose Sync now." + } else { + "Cookies removed. Future requests will use anonymous access." + } + .into(), + ); + } + Err(error) => self.cookie_error = Some(error.to_string()), + } + } + pub(super) fn settings(&self, cx: &mut Context) -> AnyElement { + let t = Tokens::current(cx); + let busy = self.cookie_busy || self.stopping.is_some(); + div().flex().flex_col().gap(px(tokens::SPACE)).max_w(px(720.)) + .child(div().flex().justify_between().items_center() + .child(div().text_size(px(tokens::PAGE_TITLE)).font_weight(FontWeight::BOLD).child("Settings")) + .child(Button::new("close-settings").ghost().label("Done").disabled(busy).on_click(cx.listener(|view, _, window, cx| { + view.settings_open = false; + view.cookie_input = cookie_input(window, cx); + view.focus.focus(window, cx); + cx.notify(); + })))) + .child(div().flex().flex_col().gap_2() + .child(div().text_size(px(tokens::TITLE)).font_weight(FontWeight::SEMIBOLD).child("YouTube")) + .child("If YouTube asks you to sign in, add cookies from a private browser session to continue importing.") + .child(div().text_color(t.muted).child("Cookies stay on this computer and are shared with the Listenbox CLI. Keep them private, like a password.")) + .child(div().flex().items_start().child(Button::new("cookie-guide").ghost().icon(assets::IconName::ExternalLink).label("How to export YouTube cookies").on_click(|_, _, cx| cx.open_url(cookies::GUIDE_URL))))) + .child(div().flex().flex_col().gap_2() + .child(div().font_weight(FontWeight::SEMIBOLD).child("YouTube cookies")) + .child(div().text_color(t.muted).child(if self.cookie_saved { "A session is saved. Pasting a new export replaces it." } else { "Paste a Netscape cookies.txt export, including the header line." })) + .child(Textarea::new(&self.cookie_input).h(px(132.)).accessibility_id("cookie-export").aria_label("YouTube cookies").disabled(busy)) + .when_some(self.cookie_error.clone(), |pane, error| pane.child(div().text_color(t.danger).child(error))) + .when_some(self.cookie_message.clone(), |pane, message| pane.child(div().child(message))) + .child(div().flex().items_center().gap_3() + .child(Button::new("save-cookies").primary().label(if busy { "Please wait…" } else { "Save cookies" }).disabled(busy).on_click(cx.listener(|view, _, _, cx| view.save_cookies(false, cx)))) + .child(Button::new("remove-cookies").ghost().label("Remove cookies").disabled(busy || (!self.cookie_saved && self.cookie_error.is_none())).on_click(cx.listener(|view, _, _, cx| view.save_cookies(true, cx)))))) + .into_any_element() + } +} diff --git a/crates/desktop/src/workspace/tests.rs b/crates/desktop/src/workspace/tests.rs index cf1eef1..5bcc2fc 100644 --- a/crates/desktop/src/workspace/tests.rs +++ b/crates/desktop/src/workspace/tests.rs @@ -629,3 +629,77 @@ fn library_filters_connections_and_shares_artwork_requests(cx: &mut TestAppConte "reload must refresh stale artwork" ); } + +#[gpui_kit::test] +async fn youtube_settings_validate_save_forget_and_link_to_guide(cx: &mut TestAppContext) { + let (_profile, handle, view) = quit_workspace(cx); + cx.update_window(handle.into(), |_, window, cx| { + view.update(cx, |view, cx| { + view.error = Some(ErrorNotice::youtube_sign_in()); + cx.notify(); + }); + window.render_frame(cx); + window.click("youtube-sign-in-settings", cx); + }) + .unwrap(); + wait_for(cx, &view, |view| !view.cookie_busy).await; + cx.update_window(handle.into(), |_, window, cx| { + window.render_frame(cx); + assert!(view.read(cx).settings_open); + window.click("cookie-guide", cx); + view.read(cx).cookie_input.clone().update(cx, |input, cx| { + input.set_value("invalid export", window, cx) + }); + window.scroll( + "cookie-guide", + ScrollDelta::Pixels(point(px(0.), px(-400.))), + cx, + ); + window.click("save-cookies", cx); + }) + .unwrap(); + wait_for(cx, &view, |view| !view.cookie_busy).await; + assert_eq!( + cx.opened_url().as_deref(), + Some(listenbox_sync_engine::cookies::GUIDE_URL) + ); + cx.update_window(handle.into(), |_, window, cx| { + assert!(view.read(cx).cookie_error.is_some()); + assert!(!view.read(cx).client.cookie_jar().is_enabled().unwrap()); + view.read(cx).cookie_input.clone().update(cx, |input, cx| input.set_value("# Netscape HTTP Cookie File\n.youtube.com\tTRUE\t/\tTRUE\t0\tSAPISID\tsynthetic-ui-cookie\n", window, cx)); + window.render_frame(cx); + window.scroll("cookie-guide", ScrollDelta::Pixels(point(px(0.), px(-400.))), cx); + window.click("save-cookies", cx); + }).unwrap(); + wait_for(cx, &view, |view| !view.cookie_busy).await; + cx.update_window(handle.into(), |_, window, cx| { + assert!(view.read(cx).client.cookie_jar().is_enabled().unwrap()); + assert!(view.read(cx).cookie_input.read(cx).value().is_empty()); + assert!(view.read(cx).cookie_error.is_none()); + window.render_frame(cx); + window.scroll( + "save-cookies", + ScrollDelta::Pixels(point(px(0.), px(600.))), + cx, + ); + window.click("close-settings", cx); + window.render_frame(cx); + window.click("open-settings", cx); + }) + .unwrap(); + wait_for(cx, &view, |view| !view.cookie_busy).await; + cx.update_window(handle.into(), |_, window, cx| { + assert!(view.read(cx).cookie_saved); + assert!(view.read(cx).cookie_input.read(cx).value().is_empty()); + window.render_frame(cx); + window.scroll( + "cookie-guide", + ScrollDelta::Pixels(point(px(0.), px(-400.))), + cx, + ); + window.click("remove-cookies", cx); + }) + .unwrap(); + wait_for(cx, &view, |view| !view.cookie_busy).await; + assert!(!cx.update(|cx| view.read(cx).client.cookie_jar().is_enabled().unwrap())); +} diff --git a/crates/sync-engine/Cargo.toml b/crates/sync-engine/Cargo.toml index 8a97972..0398a6d 100644 --- a/crates/sync-engine/Cargo.toml +++ b/crates/sync-engine/Cargo.toml @@ -6,6 +6,9 @@ license.workspace = true [dependencies] anyhow = "1" +cookie = "0.18" +cookie_store = { version = "0.22", default-features = false, features = ["serde"] } +sha1 = "0.10" ffmpeg = { package = "ffmpeg-the-third", version = "=6.0.0", default-features = false, features = ["static", "format", "software-resampling"] } chrono = { version = "0.4", default-features = false, features = ["std"] } futures-util = "0.3" diff --git a/crates/sync-engine/examples/verify_youtube.rs b/crates/sync-engine/examples/verify_youtube.rs new file mode 100644 index 0000000..ca83705 --- /dev/null +++ b/crates/sync-engine/examples/verify_youtube.rs @@ -0,0 +1,120 @@ +//! Explicit manual live probe, never invoked by tests or CI. Uses an isolated +//! profile and never publishes episodes or prints cookies or signed URLs. +use anyhow::{Context, Result, ensure}; +use listenbox_sync_engine::{ + api::Api, + config::Config, + cookies::{CookieJar, SignInRequired}, + innertube::{Playback, Stream, YouTube}, +}; +use tokio_util::sync::CancellationToken; + +#[tokio::main(flavor = "current_thread")] +async fn main() -> Result<()> { + let mut args = std::env::args().skip(1); + let url = url::Url::parse( + &args + .next() + .context("Usage: verify_youtube [cookies.txt]")?, + )?; + let playlist = url + .query_pairs() + .find(|(name, _)| name == "list") + .context("Playlist URL required")? + .1 + .into_owned(); + let profile = tempfile::tempdir()?; + if let Some(file) = args.next() { + CookieJar::new(profile.path()).import_file(std::path::Path::new(&file))?; + } + ensure!(args.next().is_none(), "Too many arguments"); + let api = Api::new( + Config::load_in(None, profile.path().into())?, + CancellationToken::new(), + )?; + let youtube = YouTube::new(&api) + .await + .context("Initialize live YouTube session")?; + let listing = youtube.snapshot(&api, &playlist).await?; + println!("Listed {} videos", listing.present.len()); + let (mut failed, mut available, mut skipped) = (0, 0, 0); + for video in listing.present { + match youtube.media(&api, &video.id).await { + Ok(Playback::Available(media)) => { + let mut allowed = true; + for stream in std::iter::once(media.video).chain(media.audio) { + if !check_stream(&api, &stream).await? { + allowed = false; + break; + } + } + if allowed { + available += 1; + println!("{}: media accessible", video.id); + } else { + skipped += 1; + println!("{}: skipped (YouTube refused media playback)", video.id); + } + } + Ok(Playback::Unavailable(_)) => { + skipped += 1; + println!("{}: skipped (YouTube marked unavailable)", video.id); + } + Err(error) => { + failed += 1; + println!( + "{}: {}", + video.id, + if error.is::() { + "sign-in required" + } else { + "extraction failed" + } + ); + } + } + } + println!("{available} accessible, {skipped} skipped, {failed} failed"); + ensure!(available > 0, "No playable videos reached the media check"); + ensure!(failed == 0, "{failed} videos failed extraction"); + Ok(()) +} + +async fn check_stream(api: &Api, stream: &Stream) -> Result { + // Check both ends, including data beyond a temporary token's initial allowance. + let mut range = "bytes=0-65535".to_owned(); + for _ in 0..2 { + let response = api + .send( + api.http + .get(&stream.url) + .header("user-agent", &stream.user_agent) + .header("range", &range), + ) + .await + .map_err(|_| anyhow::anyhow!("Media request failed"))?; + if response.status() == reqwest::StatusCode::FORBIDDEN { + return Ok(false); + } + ensure!( + response.status() == reqwest::StatusCode::PARTIAL_CONTENT, + "Media range rejected: {}", + response.status() + ); + let total = response + .headers() + .get("content-range") + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.rsplit('/').next()) + .and_then(|value| value.parse::().ok()) + .filter(|total| *total > 0) + .context("Media response has no valid total length")?; + range = format!("bytes={}-{}", total.saturating_sub(65536), total - 1); + let bytes = api + .bytes(response, 128 << 10) + .await + .map_err(|_| anyhow::anyhow!("Media request failed"))?; + ensure!(!bytes.is_empty(), "Empty media response"); + } + Ok(true) +} diff --git a/crates/sync-engine/src/api.rs b/crates/sync-engine/src/api.rs index 10bfebb..34e385c 100644 --- a/crates/sync-engine/src/api.rs +++ b/crates/sync-engine/src/api.rs @@ -41,6 +41,15 @@ impl Api { .and_then(|raw| serde_json::from_slice::(&raw).ok()) .filter(|auth| crate::config::credential_valid(&auth.api_key)) .map(|auth| auth.api_key); + Ok(Self { + config, + credential, + http: Self::http_client(false)?, + cancel, + }) + } + + pub(crate) fn http_client(no_redirects: bool) -> Result { let mut builder = Client::builder() .connect_timeout(Duration::from_secs(30)) .timeout(Duration::from_secs(30 * 60)); @@ -49,12 +58,10 @@ impl Api { builder = builder.add_root_certificate(certificate); } } - Ok(Self { - config, - credential, - http: reqwest_middleware::ClientBuilder::new(builder.build()?).build(), - cancel, - }) + if no_redirects { + builder = builder.redirect(reqwest::redirect::Policy::none()); + } + Ok(reqwest_middleware::ClientBuilder::new(builder.build()?).build()) } pub fn client(&self) -> crate::publicapi::Client { diff --git a/crates/sync-engine/src/client.rs b/crates/sync-engine/src/client.rs index cee8c4b..e2f553e 100644 --- a/crates/sync-engine/src/client.rs +++ b/crates/sync-engine/src/client.rs @@ -30,6 +30,9 @@ impl Client { config, }) } + pub fn cookie_jar(&self) -> crate::cookies::CookieJar { + crate::cookies::CookieJar::new(&self.config.directory) + } pub fn downloads(&self) -> DownloadManager { self.engine.downloads.clone() } diff --git a/crates/sync-engine/src/cookies.rs b/crates/sync-engine/src/cookies.rs new file mode 100644 index 0000000..ea0698d --- /dev/null +++ b/crates/sync-engine/src/cookies.rs @@ -0,0 +1,373 @@ +//! Private YouTube sessions shared across CLI/desktop processes. +//! +//! Readers open immutable snapshots. Writers hold a separate OS lock through +//! read/compare/merge and fsync + atomic replacement. Each import/removal starts +//! a new generation; each cookie has its own revision (including tombstones). +//! Thus late responses cannot roll back rotations, resurrect deletions, or undo +//! an explicit replacement. A crash exposes either complete file, never a prefix. +use crate::config::write_private_json; +use anyhow::{Context, Result, bail, ensure}; +use cookie_store::{Cookie, CookieDomain}; +use serde::{Deserialize, Serialize}; +use std::{ + fs::{File, OpenOptions}, + io::Read, + path::{Path, PathBuf}, +}; +use url::Url; + +pub const GUIDE_URL: &str = + "https://listenbox.app/guides/import-youtube-as-a-podcast/#when-youtube-asks-you-to-sign-in"; +const MAX_BYTES: usize = 1 << 20; +const MAX_COOKIES: usize = 1024; + +#[derive(Debug)] +pub struct SignInRequired; +impl std::fmt::Display for SignInRequired { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("YouTube needs a signed-in session. Open Settings → YouTube to paste fresh cookies, or run listenbox youtube-cookies import , then sync again.") + } +} +impl std::error::Error for SignInRequired {} + +// No Debug implementations: snapshots contain credentials. +#[derive(Clone, Serialize, Deserialize)] +struct Entry { + cookie: Cookie<'static>, + revision: String, +} +#[derive(Clone, Default, Serialize, Deserialize)] +pub(crate) struct Snapshot { + generation: String, + pub(crate) enabled: bool, + entries: Vec, +} +impl Snapshot { + pub(crate) fn header(&self, url: &Url) -> String { + if !self.enabled || url.scheme() != "https" || !url.host_str().is_some_and(youtube_domain) { + return String::new(); + } + let mut cookies: Vec<_> = self + .entries + .iter() + .map(|entry| &entry.cookie) + .filter(|cookie| !cookie.is_expired() && cookie.matches(url)) + .collect(); + cookies.sort_by(|a, b| { + b.path + .as_ref() + .len() + .cmp(&a.path.as_ref().len()) + .then(a.name().cmp(b.name())) + }); + cookies + .into_iter() + .map(|cookie| format!("{}={}", cookie.name(), cookie.value())) + .collect::>() + .join("; ") + } + fn revision(&self, cookie: &Cookie<'_>) -> Option<&str> { + self.entries + .iter() + .find(|entry| same_key(&entry.cookie, cookie)) + .map(|entry| entry.revision.as_str()) + } +} + +#[derive(Clone)] +pub struct CookieJar { + directory: PathBuf, +} +impl CookieJar { + pub fn new(profile: &Path) -> Self { + Self { + directory: profile.join("youtube-cookies"), + } + } + fn path(&self) -> PathBuf { + self.directory.join("jar.json") + } + fn write_lock(&self) -> Result { + std::fs::create_dir_all(&self.directory) + .context("Create private YouTube cookie directory")?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&self.directory, std::fs::Permissions::from_mode(0o700))?; + } + let file = OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .open(self.directory.join("write.lock")) + .context("Open YouTube cookie lock")?; + file.lock().context("Lock YouTube cookie updates")?; + Ok(file) + } + fn save(&self, saved: &Snapshot) -> Result<()> { + ensure!( + serde_json::to_vec_pretty(saved)?.len() < MAX_BYTES, + "YouTube cookie jar is too large; export a smaller YouTube session" + ); + write_private_json(&self.path(), saved).context("Save YouTube cookies") + } + pub(crate) fn snapshot(&self) -> Result { + let file = match File::open(self.path()) { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Ok(Snapshot::default()); + } + Err(error) => return Err(error).context("Read saved YouTube cookies"), + }; + let bytes = read_bounded(file)?; + let snapshot: Snapshot = serde_json::from_slice(&bytes) + .map_err(|_| anyhow::anyhow!("Saved YouTube cookies are invalid. Replace them in Settings or with youtube-cookies import."))?; + ensure!( + snapshot.entries.len() <= MAX_COOKIES + && snapshot + .entries + .iter() + .all(|entry| valid_cookie(&entry.cookie)), + "Saved YouTube cookies are invalid; import a fresh export" + ); + Ok(snapshot) + } + pub fn is_enabled(&self) -> Result { + Ok(self.snapshot()?.enabled) + } + pub fn import_file(&self, path: &Path) -> Result<()> { + let bytes = read_bounded(File::open(path).context("Open cookie export")?)?; + let text = std::str::from_utf8(&bytes).context("Cookie export must be UTF-8 text")?; + self.import(text) + } + pub fn import(&self, text: &str) -> Result<()> { + let cookies = parse_netscape(text)?; + let saved = Snapshot { + generation: revision(), + enabled: true, + entries: cookies + .into_iter() + .map(|cookie| Entry { + cookie, + revision: revision(), + }) + .collect(), + }; + let _lock = self.write_lock()?; + self.save(&saved) + } + pub fn remove(&self) -> Result<()> { + let _lock = self.write_lock()?; + // Persist the new generation, so already-running requests cannot restore it. + self.save(&Snapshot { + generation: revision(), + ..Snapshot::default() + }) + .context("Remove YouTube cookies") + } + pub(crate) fn update(&self, base: &Snapshot, url: &Url, headers: &[String]) -> Result<()> { + if !base.enabled + || headers.is_empty() + || url.scheme() != "https" + || !url.host_str().is_some_and(youtube_domain) + { + return Ok(()); + } + let mut updates = Vec::new(); + for header in headers.iter().take(MAX_COOKIES) { + if header.len() > 16 << 10 { + continue; + } + let Ok(cookie) = Cookie::parse(header.clone(), url).map(Cookie::into_owned) else { + continue; + }; + if valid_cookie(&cookie) { + // Last Set-Cookie for a key in the same response wins. + updates.retain(|previous| !same_key(previous, &cookie)); + updates.push(cookie); + } + } + if updates.is_empty() { + return Ok(()); + } + let _lock = self.write_lock()?; + let mut current = self.snapshot()?; + if current.generation != base.generation || !current.enabled { + return Ok(()); + } + let mut changed = false; + for cookie in updates { + if current.revision(&cookie) != base.revision(&cookie) { + continue; + } + if let Some(entry) = current + .entries + .iter_mut() + .find(|entry| same_key(&entry.cookie, &cookie)) + { + *entry = Entry { + cookie, + revision: revision(), + }; + } else { + ensure!( + current.entries.len() < MAX_COOKIES, + "YouTube cookie jar is full; import a fresh export" + ); + current.entries.push(Entry { + cookie, + revision: revision(), + }); + } + changed = true; + } + if changed { + self.save(¤t)?; + } + Ok(()) + } +} +fn revision() -> String { + uuid::Uuid::new_v4().to_string() +} +fn read_bounded(file: File) -> Result> { + let mut bytes = Vec::new(); + file.take((MAX_BYTES + 1) as u64) + .read_to_end(&mut bytes) + .context("Read YouTube cookies")?; + ensure!( + bytes.len() <= MAX_BYTES, + "Cookie file is too large (maximum 1 MiB)" + ); + Ok(bytes) +} +fn youtube_domain(domain: &str) -> bool { + domain == "youtube.com" || domain.ends_with(".youtube.com") +} +fn domain<'a>(cookie: &'a Cookie<'_>) -> &'a str { + match &cookie.domain { + CookieDomain::HostOnly(host) | CookieDomain::Suffix(host) => host, + _ => "", + } +} +fn same_key(a: &Cookie<'_>, b: &Cookie<'_>) -> bool { + domain(a) == domain(b) && a.path.as_ref() == b.path.as_ref() && a.name() == b.name() +} +fn valid_cookie(cookie: &Cookie<'_>) -> bool { + youtube_domain(domain(cookie)) + && valid_pair(cookie.name(), cookie.value()) + && (!cookie.name().starts_with("__Secure-") || cookie.secure() == Some(true)) + && (!cookie.name().starts_with("__Host-") + || (cookie.secure() == Some(true) + && cookie.path.as_ref() == "/" + && matches!(cookie.domain, CookieDomain::HostOnly(_)))) +} +fn valid_pair(name: &str, value: &str) -> bool { + !name.is_empty() + && name + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b"!#$%&'*+-.^_`|~".contains(&b)) + && value + .bytes() + .all(|b| (0x21..=0x7e).contains(&b) && !b"\";,\\".contains(&b)) +} +fn parse_netscape(text: &str) -> Result>> { + ensure!( + text.len() <= MAX_BYTES, + "Cookie export is too large (maximum 1 MiB)" + ); + let mut lines = text.trim_start_matches('\u{feff}').lines(); + ensure!( + matches!( + lines.next().map(str::trim), + Some("# Netscape HTTP Cookie File" | "# HTTP Cookie File") + ), + "Paste the entire Netscape cookies.txt export, including its header. JSON is not supported." + ); + let mut cookies = Vec::new(); + for (index, line) in lines.enumerate() { + let (line, http_only) = match line.strip_prefix("#HttpOnly_") { + Some(line) => (line, true), + None if line.starts_with('#') || line.trim().is_empty() => continue, + None => (line, false), + }; + let fields: Vec<_> = line.split('\t').collect(); + let invalid = || { + anyhow::anyhow!( + "Invalid cookies.txt row {}. Export cookies in Netscape format again.", + index + 2 + ) + }; + if fields.len() != 7 { + return Err(invalid()); + } + let [host, subdomains, path, secure, expires, name, value] = fields.as_slice() else { + unreachable!() + }; + let host = host.trim_start_matches('.').to_ascii_lowercase(); + if !youtube_domain(&host) { + continue; + } + if !matches!(*subdomains, "TRUE" | "FALSE") + || !matches!(*secure, "TRUE" | "FALSE") + || !valid_pair(name, value) + || !path.starts_with('/') + || path.bytes().any(|b| b <= 0x20 || b >= 0x7f || b == b';') + { + return Err(invalid()); + } + let expires: i64 = expires.parse().map_err(|_| invalid())?; + if expires < 0 { + return Err(invalid()); + } + if !host + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b".-".contains(&b)) + { + return Err(invalid()); + } + let url = Url::parse(&format!("https://{host}/")).map_err(|_| invalid())?; + let mut raw = cookie::Cookie::new((*name).to_owned(), (*value).to_owned()); + raw.set_path((*path).to_owned()); + raw.set_secure(*secure == "TRUE"); + raw.set_http_only(http_only); + if *subdomains == "TRUE" { + raw.set_domain(host); + } + if expires != 0 { + raw.set_expires( + cookie::time::OffsetDateTime::from_unix_timestamp(expires) + .map_err(|_| invalid())?, + ); + } + let cookie = Cookie::try_from_raw_cookie(&raw, &url) + .map_err(|_| invalid())? + .into_owned(); + if !valid_cookie(&cookie) { + return Err(invalid()); + } + if cookie.is_expired() { + continue; + } + if cookies.iter().any(|previous| same_key(previous, &cookie)) { + bail!( + "Duplicate cookie in row {}; export cookies again", + index + 2 + ); + } + cookies.push(cookie); + ensure!( + cookies.len() <= MAX_COOKIES, + "Too many YouTube cookies in this export" + ); + } + ensure!( + !cookies.is_empty(), + "No unexpired YouTube cookies found. Export a fresh YouTube session." + ); + Ok(cookies) +} + +#[cfg(test)] +mod tests; diff --git a/crates/sync-engine/src/cookies/tests.rs b/crates/sync-engine/src/cookies/tests.rs new file mode 100644 index 0000000..8e5634a --- /dev/null +++ b/crates/sync-engine/src/cookies/tests.rs @@ -0,0 +1,267 @@ +use super::*; +use std::sync::{Arc, Barrier}; + +fn export(value: &str) -> String { + format!("# Netscape HTTP Cookie File\n.youtube.com\tTRUE\t/\tTRUE\t0\tSAPISID\t{value}\n") +} +fn url() -> Url { + Url::parse("https://www.youtube.com/youtubei/v1/player").unwrap() +} +fn fixture() -> (tempfile::TempDir, CookieJar) { + let directory = tempfile::tempdir().unwrap(); + let jar = CookieJar::new(directory.path()); + (directory, jar) +} + +#[test] +fn validates_exports_without_disclosing_values_or_replacing_good_state() { + let (_dir, jar) = fixture(); + jar.import(&export("private-value").replace('\n', "\r\n")) + .unwrap(); + for input in [ + "[{\"value\":\"private-value\"}]".into(), + export("private-value; other=oops"), + export("private-value\rinjected"), + export("private-value").replace("TRUE", "maybe"), + ] { + let message = jar.import(&input).unwrap_err().to_string(); + assert!(!message.contains("private-value")); + assert_eq!( + jar.snapshot().unwrap().header(&url()), + "SAPISID=private-value" + ); + } + assert!(jar.import(&"x".repeat(MAX_BYTES + 1)).is_err()); +} + +#[test] +fn respects_host_path_secure_expiry_and_http_only() { + let (_dir, jar) = fixture(); + let input = concat!( + "# HTTP Cookie File\n", + "#HttpOnly_.youtube.com\tTRUE\t/\tTRUE\t0\tSID\tsecure\n", + "www.youtube.com\tFALSE\t/youtubei\tFALSE\t0\tHOST\tonly\n", + ".youtube.com\tTRUE\t/\tTRUE\t1\tOLD\texpired\n", + ".example.com\tTRUE\t/\tTRUE\t0\tOTHER\tignored,elsewhere\n" + ); + jar.import(input).unwrap(); + let snapshot = jar.snapshot().unwrap(); + assert_eq!(snapshot.header(&url()), "HOST=only; SID=secure"); + assert_eq!( + snapshot.header(&Url::parse("https://m.youtube.com/").unwrap()), + "SID=secure" + ); + assert_eq!( + snapshot.header(&Url::parse("https://www.youtube.com/youtubeix").unwrap()), + "SID=secure" + ); + assert!( + snapshot + .header(&Url::parse("http://m.youtube.com/").unwrap()) + .is_empty() + ); + assert!( + snapshot + .header(&Url::parse("https://youtube.com.example.com/").unwrap()) + .is_empty() + ); + assert!( + snapshot + .header(&Url::parse("https://media.googlevideo.com/").unwrap()) + .is_empty() + ); +} + +#[test] +fn merges_concurrent_updates_without_stale_overwrites_or_resurrection() { + let (_dir, jar) = fixture(); + jar.import(&export("seed")).unwrap(); + let initial = jar.snapshot().unwrap(); + let gate = Arc::new(Barrier::new(3)); + std::thread::scope(|scope| { + for header in [ + "FIRST=one; Domain=youtube.com; Path=/; Secure", + "SECOND=two; Domain=youtube.com; Path=/; Secure", + ] { + let (jar, initial, gate) = (jar.clone(), initial.clone(), gate.clone()); + scope.spawn(move || { + gate.wait(); + jar.update(&initial, &url(), &[header.into()]).unwrap(); + }); + } + gate.wait(); + }); + let both = jar.snapshot().unwrap().header(&url()); + assert!(both.contains("FIRST=one") && both.contains("SECOND=two")); + jar.update( + &initial, + &url(), + &["SAPISID=new; Domain=youtube.com; Path=/; Secure".into()], + ) + .unwrap(); + jar.update( + &initial, + &url(), + &["SAPISID=stale; Domain=youtube.com; Path=/; Secure".into()], + ) + .unwrap(); + assert!( + jar.snapshot() + .unwrap() + .header(&url()) + .contains("SAPISID=new") + ); + let before_delete = jar.snapshot().unwrap(); + jar.update( + &before_delete, + &url(), + &["SAPISID=; Max-Age=0; Domain=youtube.com; Path=/; Secure".into()], + ) + .unwrap(); + jar.update( + &before_delete, + &url(), + &["SAPISID=resurrected; Domain=youtube.com; Path=/; Secure".into()], + ) + .unwrap(); + assert!(!jar.snapshot().unwrap().header(&url()).contains("SAPISID=")); + jar.import(&export("replacement")).unwrap(); + jar.update( + &initial, + &url(), + &["SAPISID=late; Domain=youtube.com; Path=/; Secure".into()], + ) + .unwrap(); + assert_eq!( + jar.snapshot().unwrap().header(&url()), + "SAPISID=replacement" + ); + let before_remove = jar.snapshot().unwrap(); + jar.remove().unwrap(); + jar.update( + &before_remove, + &url(), + &["SAPISID=late; Domain=youtube.com; Path=/; Secure".into()], + ) + .unwrap(); + assert!(!jar.snapshot().unwrap().enabled); +} + +#[test] +fn readers_do_not_wait_for_writer_and_ignore_uncommitted_temporary_files() { + let (directory, jar) = fixture(); + jar.import(&export("committed")).unwrap(); + let _lock = jar.write_lock().unwrap(); + std::fs::write(directory.path().join("youtube-cookies/incomplete.tmp"), "{").unwrap(); + let (send, receive) = std::sync::mpsc::channel(); + let reader = jar.clone(); + let thread = std::thread::spawn(move || { + send.send(reader.snapshot().unwrap().header(&url())) + .unwrap() + }); + assert_eq!( + receive + .recv_timeout(std::time::Duration::from_secs(2)) + .unwrap(), + "SAPISID=committed" + ); + thread.join().unwrap(); +} + +#[test] +fn rejects_foreign_response_cookies_and_persists_rotation_across_reopen() { + let (directory, jar) = fixture(); + jar.import(&export("seed")).unwrap(); + let snapshot = jar.snapshot().unwrap(); + jar.update( + &snapshot, + &url(), + &[ + "SAPISID=rotated; Domain=youtube.com; Path=/; Secure; Max-Age=3600".into(), + "FOREIGN=no; Domain=com; Path=/".into(), + "OTHER=no; Domain=example.com; Path=/".into(), + ], + ) + .unwrap(); + let reopened = CookieJar::new(directory.path()).snapshot().unwrap(); + assert_eq!(reopened.header(&url()), "SAPISID=rotated"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + assert_eq!( + std::fs::metadata(jar.path()).unwrap().permissions().mode() & 0o777, + 0o600 + ); + } +} + +#[test] +fn process_writer() { + let Some(profile) = std::env::var_os("LISTENBOX_COOKIE_TEST_PROFILE") else { + return; + }; + let jar = CookieJar::new(Path::new(&profile)); + let base = jar.snapshot().unwrap(); + use std::io::Write; + println!("COOKIE_WRITER_READY"); + std::io::stdout().flush().unwrap(); + let mut gate = String::new(); + std::io::stdin().read_line(&mut gate).unwrap(); + assert_eq!(gate.trim(), "commit"); + jar.update( + &base, + &url(), + &[ + "SAPISID=stale-process; Domain=youtube.com; Path=/; Secure".into(), + "PROCESS=merged; Domain=youtube.com; Path=/; Secure".into(), + ], + ) + .unwrap(); +} + +#[test] +fn process_updates_compare_against_committed_revisions() { + use std::{ + io::{BufRead, Write}, + process::{Command, Stdio}, + }; + let (directory, jar) = fixture(); + jar.import(&export("seed")).unwrap(); + let mut child = Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "cookies::tests::process_writer", "--nocapture"]) + .env("LISTENBOX_COOKIE_TEST_PROFILE", directory.path()) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + let stdout = child.stdout.take().unwrap(); + let (ready, received) = std::sync::mpsc::channel(); + let reader = std::thread::spawn(move || { + for line in std::io::BufReader::new(stdout).lines() { + if line.unwrap() == "COOKIE_WRITER_READY" { + ready.send(()).unwrap(); + } + } + }); + if received + .recv_timeout(std::time::Duration::from_secs(3)) + .is_err() + { + let _ = child.kill(); + let _ = child.wait(); + panic!("child did not establish a cookie snapshot"); + } + let base = jar.snapshot().unwrap(); + jar.update( + &base, + &url(), + &["SAPISID=new-process; Domain=youtube.com; Path=/; Secure".into()], + ) + .unwrap(); + child.stdin.take().unwrap().write_all(b"commit\n").unwrap(); + assert!(child.wait().unwrap().success()); + reader.join().unwrap(); + let header = jar.snapshot().unwrap().header(&url()); + assert!(header.contains("SAPISID=new-process") && header.contains("PROCESS=merged")); + assert!(!header.contains("stale-process")); +} diff --git a/crates/sync-engine/src/download.rs b/crates/sync-engine/src/download.rs index 9e64fbe..8f19ee6 100644 --- a/crates/sync-engine/src/download.rs +++ b/crates/sync-engine/src/download.rs @@ -10,6 +10,16 @@ use sha2::{Digest, Sha256}; use std::{io::SeekFrom, path::Path}; use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt}; +#[derive(Debug)] +pub(crate) struct MediaUnavailable; + +impl std::fmt::Display for MediaUnavailable { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("YouTube did not allow playback of this video. It will be checked again at the next sync.") + } +} +impl std::error::Error for MediaUnavailable {} + pub(crate) async fn download( api: &Api, source: &Stream, @@ -26,6 +36,9 @@ pub(crate) async fn download( .header("User-Agent", &source.user_agent), ) .await?; + if response.status() == reqwest::StatusCode::FORBIDDEN { + return Err(MediaUnavailable.into()); + } ensure!( response.status() == reqwest::StatusCode::PARTIAL_CONTENT, "Resolve download length: HTTP {}", @@ -119,6 +132,9 @@ pub(crate) async fn download( request = request.header("If-Range", validator); } let mut response = api.send(request).await?; + if response.status() == reqwest::StatusCode::FORBIDDEN { + return Err(MediaUnavailable.into()); + } ensure!( response.status() == reqwest::StatusCode::PARTIAL_CONTENT, "Download range returned HTTP {}; source may have changed", diff --git a/crates/sync-engine/src/downloads.rs b/crates/sync-engine/src/downloads.rs index d591c38..3ecddb6 100644 --- a/crates/sync-engine/src/downloads.rs +++ b/crates/sync-engine/src/downloads.rs @@ -390,7 +390,9 @@ impl Transfer { } pub fn error(&self, error: &anyhow::Error) { - self.update(|item| item.error = Some(crate::redact(&format!("{error:#}")))); + self.update(|item| item.error = Some(if error.is::() { + "YouTube needs a signed-in session. Open Settings → YouTube to add fresh cookies, then sync again.".into() + } else { crate::redact(&format!("{error:#}")) })); } pub fn start_download(&self, total: u64, chunk_bytes: usize) { diff --git a/crates/sync-engine/src/innertube.rs b/crates/sync-engine/src/innertube.rs index 72ffa0d..b076b64 100644 --- a/crates/sync-engine/src/innertube.rs +++ b/crates/sync-engine/src/innertube.rs @@ -1,6 +1,8 @@ //! Application policy over the shared typed youtubei bindings. use crate::api::{Api, read_bounded}; +use crate::cookies::{CookieJar, SignInRequired}; use anyhow::{Context, Result, bail, ensure}; +use sha1::{Digest, Sha1}; use std::collections::HashSet; use youtubei::{ BrowseOptions, Client, Engine, EngineOptions, FetchRequest, FetchResponse, Format, @@ -13,7 +15,7 @@ use youtubei::{ pub struct YouTube { client: Innertube, parse_failed: std::rc::Rc>, - playback_client: Client, + user_agent: String, } pub struct PlaylistSnapshot { @@ -55,7 +57,10 @@ pub struct Stream { impl YouTube { pub async fn new(api: &Api) -> Result { - let playback_client = Client::VisionOs; + let jar = CookieJar::new(&api.config.directory); + let snapshot_jar = jar.clone(); + let initial = tokio::task::spawn_blocking(move || snapshot_jar.snapshot()).await??; + let cookie = initial.header(&url::Url::parse("https://www.youtube.com/")?); let engine = Engine::with_options(EngineOptions::default()).await?; let cancel = api.cancel.clone(); engine @@ -67,7 +72,7 @@ impl YouTube { .value_with(|ctx| { Ok(youtubei::rquickjs::Function::new( ctx, - move |_error: youtubei::rquickjs::Value<'_>| { + move |_: youtubei::rquickjs::Object<'_>| { failed.set(true); }, )? @@ -79,11 +84,14 @@ impl YouTube { .await? .call("setParserErrorHandler", &[callback.into()]) .await?; - let api = api.clone(); + let mut api = api.clone(); + // Never forward browser credentials through a redirect. + api.http = Api::http_client(true)?; let continuations = std::sync::Arc::new(parking_lot::Mutex::new(HashSet::new())); let fetch = engine .fetch_with(move |request| { let api = api.clone(); + let jar = jar.clone(); let continuations = continuations.clone(); async move { if request.url.contains("/youtubei/v1/browse") @@ -96,7 +104,7 @@ impl YouTube { { return Err(youtubei::Error::new("Repeated YouTube continuation")); } - fetch(&api, request) + fetch(&api, &jar, request) .await .map_err(|error| youtubei::Error::new(error.to_string())) } @@ -110,19 +118,22 @@ impl YouTube { location: Some("US".into()), cache: Some(cache.as_cache()), fetch: Some(fetch), + cookie: (!cookie.is_empty()).then_some(cookie), generate_session_locally: Some(false), fail_fast: Some(true), - retrieve_player: Some(false), + // Web playback needs the player signature timestamp. + retrieve_player: Some(true), retrieve_innertube_config: Some(false), ..Default::default() }, ) .await .context("initialize YouTube")?; + let user_agent = client.session().await?.user_agent().await?; Ok(Self { client, parse_failed, - playback_client, + user_agent, }) } @@ -264,7 +275,7 @@ impl YouTube { .get_basic_info( id, GetVideoInfoOptions { - client: Some(self.playback_client), + client: Some(Client::Web), ..Default::default() }, ) @@ -279,7 +290,14 @@ impl YouTube { .as_ref() .is_some_and(|info| info["status"] == "ERROR") => { - return Ok(Playback::Unavailable(error.info.as_ref().and_then(|info| info["reason"].as_str()).unwrap_or("Video unavailable").to_owned())); + return Ok(Playback::Unavailable( + error + .info + .as_ref() + .and_then(|info| info["reason"].as_str()) + .unwrap_or("Video unavailable") + .to_owned(), + )); } Err(error) => return Err(error.into()), }; @@ -289,13 +307,33 @@ impl YouTube { .as_ref() .context("YouTube player response missing playability status")?; match status.status.as_str() { - "OK" => {}, - "UNPLAYABLE" => return Ok(Playback::Unavailable(status.reason.clone().unwrap_or_else(|| "Video unavailable".into()))), - "LOGIN_REQUIRED" => bail!("YouTube rejected anonymous playback for {id}: {}. Check YouTube access on your current network before syncing again", status.reason.as_deref().unwrap_or("Sign in required")), - _ => bail!("YouTube could not resolve {id} ({}): {}", status.status, status.reason.as_deref().unwrap_or("No reason supplied")), + "OK" => {} + "UNPLAYABLE" => { + return Ok(Playback::Unavailable( + status + .reason + .clone() + .unwrap_or_else(|| "Video unavailable".into()), + )); + } + "LOGIN_REQUIRED" => { + return Err(anyhow::Error::new(SignInRequired).context(format!( + "{} ({id})", + status.reason.as_deref().unwrap_or("Sign in required") + ))); + } + _ => bail!( + "YouTube could not resolve {id} ({}): {}", + status.status, + status.reason.as_deref().unwrap_or("No reason supplied") + ), } - if data.basic_info.is_live.unwrap_or(false) || data.basic_info.is_upcoming.unwrap_or(false) { - return Ok(Playback::Unavailable("Live or upcoming video; sync after it has finished".into())); + if data.basic_info.is_live.unwrap_or(false) + || data.basic_info.is_upcoming.unwrap_or(false) + { + return Ok(Playback::Unavailable( + "Live or upcoming video; sync after it has finished".into(), + )); } let mut formats = info.formats().await?; formats.extend(info.adaptive_formats().await?); @@ -332,14 +370,11 @@ impl YouTube { .context("YouTube video has no audio stream")?, ) }; - // VISIONOS supplies downloadable streams but omits publication dates. - // WEB still supplies that metadata when its own playback is unavailable. - let metadata = self.client.get_basic_info(id, GetVideoInfoOptions { - client: Some(Client::Web), - ..Default::default() - }).await?.data().await?; - ensure!(metadata.basic_info.id.as_deref() == Some(id), "YouTube metadata video ID differs from the requested video"); - let published = match metadata.microformat { + ensure!( + data.basic_info.id.as_deref() == Some(id), + "YouTube metadata video ID differs from the requested video" + ); + let published = match data.microformat { Some(Microformat::PlayerMicroformat(metadata)) => metadata .publish_date .filter(|date| !date.is_empty()) @@ -399,16 +434,12 @@ impl YouTube { Ok(Stream { identity: format!("{}:{}:{:?}", info.itag, info.mime_type, info.content_length), url: url.into(), - user_agent: self - .playback_client - .user_agent(self.client.engine()) - .await? - .unwrap_or_else(|| "Mozilla/5.0".into()), + user_agent: self.user_agent.clone(), }) } } -async fn fetch(api: &Api, input: FetchRequest) -> Result { +async fn fetch(api: &Api, jar: &CookieJar, input: FetchRequest) -> Result { let url = url::Url::parse(&input.url)?; let host = url.host_str().unwrap_or("").to_owned(); ensure!( @@ -424,20 +455,54 @@ async fn fetch(api: &Api, input: FetchRequest) -> Result { .any(|base| host == *base || host.ends_with(&format!(".{base}"))), "unexpected YouTube API host" ); - let mut request = api.http.request(input.method.parse()?, url); + let snapshot_jar = jar.clone(); + let snapshot = tokio::task::spawn_blocking(move || snapshot_jar.snapshot()).await??; + let cookie = snapshot.header(&url); + let mut request = api + .http + .request(input.method.parse()?, url.clone()) + .timeout(std::time::Duration::from_secs(30)); for (name, value) in input.headers { let name_lower = name.to_ascii_lowercase(); - if matches!(name_lower.as_str(), "cookie" | "authorization") && host != "www.youtube.com" { + if matches!( + name_lower.as_str(), + "cookie" | "authorization" | "x-goog-authuser" | "x-goog-pageid" + ) { continue; } if !["host", "content-length"].contains(&name_lower.as_str()) { request = request.header(name, value); } } + if !cookie.is_empty() { + request = request.header("cookie", &cookie); + if host == "www.youtube.com" && url.path().starts_with("/youtubei/") { + let timestamp = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH)? + .as_secs(); + let authorization = cookie_authorization(&cookie, timestamp); + if !authorization.is_empty() { + request = request + .header("authorization", authorization) + .header("x-origin", "https://www.youtube.com") + .header("x-goog-authuser", "0"); + } + } + } if let Some(body) = input.body { request = request.body(body); } let response = api.send(request).await?; + let updates = response + .headers() + .get_all(reqwest::header::SET_COOKIE) + .iter() + .filter_map(|value| value.to_str().ok().map(str::to_owned)) + .collect::>(); + let update_jar = jar.clone(); + // Commit response cookies before reading the body. Cancellation or a body + // failure must not discard a rotation that YouTube already performed. + tokio::task::spawn_blocking(move || update_jar.update(&snapshot, &url, &updates)).await??; let status = response.status().as_u16(); let headers = response .headers() @@ -497,3 +562,32 @@ fn duration_badge_seconds(text: &str) -> Option { } Some(total) } + +fn cookie_authorization(header: &str, timestamp: u64) -> String { + let value = |name: &str| { + header.split("; ").find_map(|part| { + part.split_once('=') + .filter(|(key, _)| *key == name) + .map(|(_, value)| value) + }) + }; + [ + ( + "SAPISIDHASH", + value("SAPISID").or_else(|| value("__Secure-3PAPISID")), + ), + ("SAPISID1PHASH", value("__Secure-1PAPISID")), + ("SAPISID3PHASH", value("__Secure-3PAPISID")), + ] + .into_iter() + .filter_map(|(scheme, sid)| { + sid.map(|sid| { + let digest = hex::encode(Sha1::digest(format!( + "{timestamp} {sid} https://www.youtube.com" + ))); + format!("{scheme} {timestamp}_{digest}") + }) + }) + .collect::>() + .join(" ") +} diff --git a/crates/sync-engine/src/lib.rs b/crates/sync-engine/src/lib.rs index 2733ec0..85089cb 100644 --- a/crates/sync-engine/src/lib.rs +++ b/crates/sync-engine/src/lib.rs @@ -6,6 +6,7 @@ pub mod auth; #[rustfmt::skip] mod clientconfig; pub mod config; +pub mod cookies; pub mod episodes; pub mod events; pub mod innertube; diff --git a/crates/sync-engine/src/sync.rs b/crates/sync-engine/src/sync.rs index ca5da04..8b95d04 100644 --- a/crates/sync-engine/src/sync.rs +++ b/crates/sync-engine/src/sync.rs @@ -269,15 +269,26 @@ impl Engine { match result { Ok(ImportOutcome::Published) => report.added += 1, Ok(ImportOutcome::Skipped(_)) => report.skipped += 1, - Err(error) => failures.push(format!("{error:#}")), + Err(error) => failures.push(error), } } if !failures.is_empty() { - bail!( + let message = format!( "{} transfer(s) failed: {}", failures.len(), - failures.join("; ") + failures + .iter() + .map(|error| format!("{error:#}")) + .collect::>() + .join("; ") ); + if failures + .iter() + .any(|error| error.is::()) + { + return Err(anyhow::Error::new(crate::cookies::SignInRequired).context(message)); + } + bail!(message); } for episode in &before.episodes { diff --git a/crates/sync-engine/src/youtube.rs b/crates/sync-engine/src/youtube.rs index a16e53d..b0a4f3d 100644 --- a/crates/sync-engine/src/youtube.rs +++ b/crates/sync-engine/src/youtube.rs @@ -227,34 +227,44 @@ pub(crate) async fn import_video( transfer.title(&media.title); transfer.duration(media.duration_seconds); } - if audio { - download( - api, - media.audio.as_ref().unwrap_or(&media.video), - &directory.join("source-audio"), - transfer, - Some((journal, operation_id.as_str())), - ) - .await?; - } else { - download( - api, - &media.video, - &directory.join("source-video"), - transfer, - Some((journal, operation_id.as_str())), - ) - .await?; - if let Some(stream) = &media.audio { + let downloaded: Result<()> = async { + if audio { download( api, - stream, + media.audio.as_ref().unwrap_or(&media.video), &directory.join("source-audio"), transfer, Some((journal, operation_id.as_str())), ) .await?; + } else { + download( + api, + &media.video, + &directory.join("source-video"), + transfer, + Some((journal, operation_id.as_str())), + ) + .await?; + if let Some(stream) = &media.audio { + download( + api, + stream, + &directory.join("source-audio"), + transfer, + Some((journal, operation_id.as_str())), + ) + .await?; + } + } + Ok(()) + } + .await; + if let Err(error) = downloaded { + if error.is::() { + return Ok(ImportOutcome::Skipped(error.to_string())); } + return Err(error); } if let Some(transfer) = transfer { transfer.phase(crate::downloads::Phase::Preparing); diff --git a/crates/youtubei/Cargo.toml b/crates/youtubei/Cargo.toml index 80f9b7d..a8b91b7 100644 --- a/crates/youtubei/Cargo.toml +++ b/crates/youtubei/Cargo.toml @@ -5,13 +5,7 @@ edition = "2024" description = "Persistent Rust bindings to youtubei.js on embedded QuickJS" license = "MIT" repository = "https://github.com/listenbox/client" -include = ["/src/**", "/build.rs", "/examples/**", "/README.md", "/LICENSE", "/THIRD-PARTY-NOTICES.txt", "/Cargo.toml", "/Cargo.lock"] - -[package.metadata.youtubei] -version = "18.1.0" -archive-sha256 = "e42d35258a29eadff84b2b9bf1f3a7b4d56f037bd5243cb4c9fc6fbb7cf3d387" -bundle-sha256 = "0125535bab0cd963dde0994bade8c1b0e719ff03edb4072a83bab8a14a2e38ae" - +publish = false [dependencies] futures-util = { version = "0.3", default-features = false, features = ["std"] } @@ -27,10 +21,3 @@ tokio = { version = "1", features = ["rt", "time", "sync", "net", "macros"] } [dev-dependencies] tokio = { version = "1", features = ["rt", "macros", "time", "sync", "net", "io-util"] } static_assertions = "1" - -[build-dependencies] -flate2 = "1" -reqwest = { version = "0.12", default-features = false, features = ["blocking", "rustls-tls-native-roots"] } -sha2 = "0.10" -tar = "0.4" -toml = "0.8" diff --git a/crates/youtubei/README.md b/crates/youtubei/README.md index 81a3381..faee64d 100644 --- a/crates/youtubei/README.md +++ b/crates/youtubei/README.md @@ -1,35 +1,34 @@ # youtubei -The client workspace's Rust crate embedding QuickJS and youtubei.js from npm. One `Innertube` -owns a reusable JavaScript instance; returned objects retain the same engine. -No application TypeScript or JavaScript bridge runs inside it. +The client workspace's Rust crate embeds QuickJS, our pinned +[YouTube.js fork](https://github.com/listenbox/YouTube.js). Live objects +retain a reusable engine; the sync engine owns cookies, transport, downloads, and +application policy. -Extracted from [Mazit](https://github.com/meoyawn/mazit/tree/fa0cd7c9edc3b0d25d91fd1bb93ef0f9bd58eb47/youtubei), preserving -its MIT attribution. Build prerequisites are Rust, a C toolchain, and CMake. -Cargo downloads the published `youtubei.js` 18.1.0 archive from the npm registry, -verifies its SHA-256, reads `package/bundle/cf-worker.js`, and verifies that file's -SHA-256 before embedding it. No JavaScript package manager, runtime, or bundler is -needed. The upstream bundle is used unchanged, including its preserved names. - -From the repository root, Moon supplies CMake through pkgx: +Extracted from [Mazit](https://github.com/meoyawn/mazit/tree/fa0cd7c9edc3b0d25d91fd1bb93ef0f9bd58eb47/youtubei), preserving its MIT attribution. +Source builds require Rust, a C toolchain, CMake, Node.js 26, Aube and Moon. +Initialize submodules recursively, then run: ```sh +git submodule update --init --recursive moon run client-youtubei:build client-youtubei:test ``` +Moon bundles `vendor/youtubejs/src/platform/cf-worker.ts` with preserved names +using upstream’s unchanged `package-lock.json`. The client owns the build tasks and +deterministic regression tests in `vendor/`; the submodule contains only upstream +source fixes. Moon writes `vendor/bundle/cf-worker.js`, which Cargo copies into its +`OUT_DIR` and embeds without a network download. Both standalone +and parent-workspace builds use these same prerequisites. End users need no +JavaScript runtime. This crate is workspace-only; its source dependency is pinned +by the nested Git submodule. License texts are in `THIRD-PARTY-NOTICES.txt`. + Sibling crates consume the checked-in source directly: ```toml youtubei = { path = "../youtubei" } ``` -The upstream version and both hashes live in `package.metadata.youtubei` in -Cargo.toml. A cold build needs access to the npm registry. Verified generated -output stays in Cargo's build-specific `OUT_DIR`; it is reused while its checksum -matches. Generated files are gitignored and excluded from the Cargo package. -Concurrent consumers never write to the shared Git checkout. Upstream license -texts are retained in `THIRD-PARTY-NOTICES.txt`. - ```rust use youtubei::{Innertube, SessionOptions, models::PlaylistItem}; @@ -149,10 +148,10 @@ remain on the app's existing worker pool after receiving owned URL/metadata. The crate embeds rquickjs 0.11 with LLRT 0.8.1-beta's Rust implementations of fetch, streams, URL, events, timers, encoding, and crypto. Rust supplies `Platform.load`, player evaluation, structured cloning, and caching. The complete -published `bundle/cf-worker.js` is loaded unchanged; Rust replaces its platform shim +source-built `vendor/bundle/cf-worker.js` is loaded; Rust replaces its platform shim after evaluation. `--keep-names` is required by the upstream parser. -The published bundle imports `module.createRequire` to probe optional worker +The source bundle imports `module.createRequire` to probe optional worker threads. The crate supplies a native-only require factory because LLRT 0.8.1 exports `require` directly under that name. Native module loads work; filesystem CommonJS loads and worker threads are unavailable and raise JavaScript errors. @@ -160,8 +159,8 @@ The bundle's own optional-worker handling catches that error. Tests run the real bundle offline: object lifetime/identity, independent workers, overlapping promises, Rust callbacks, errors, flat continuation pages, VISIONOS -request construction, parser nodes, formats, and the platform hooks. They do not -establish compatibility of every upstream feature (e.g. account authentication). +request construction, parser nodes, formats, and the platform hooks. +These tests do not establish compatibility of every upstream feature. LLRT implements a subset of browser APIs. Cancelling a Rust wait does not itself abort a JavaScript operation; use the upstream AbortSignal/cancellation API when needed, and drive background jobs with `Engine::idle` while using subscriptions. diff --git a/crates/youtubei/THIRD-PARTY-NOTICES.txt b/crates/youtubei/THIRD-PARTY-NOTICES.txt index 58535c9..b3fe6ab 100644 --- a/crates/youtubei/THIRD-PARTY-NOTICES.txt +++ b/crates/youtubei/THIRD-PARTY-NOTICES.txt @@ -289,4 +289,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file +SOFTWARE. diff --git a/crates/youtubei/build.rs b/crates/youtubei/build.rs index d7b71e5..03bb8f3 100644 --- a/crates/youtubei/build.rs +++ b/crates/youtubei/build.rs @@ -1,57 +1,14 @@ -use flate2::read::GzDecoder; -use sha2::{Digest, Sha256}; -use std::{env, error::Error, fs, io::Read, path::PathBuf, time::Duration}; +use std::{env, error::Error, fs, path::PathBuf}; fn main() -> Result<(), Box> { - println!("cargo:rerun-if-changed=Cargo.toml"); - let manifest: toml::Value = toml::from_str(&fs::read_to_string("Cargo.toml")?)?; - let upstream = &manifest["package"]["metadata"]["youtubei"]; - let field = |name: &str| { - upstream[name] - .as_str() - .expect("invalid youtubei package metadata") - }; + let bundle = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()) + .join("../../vendor/bundle/cf-worker.js"); + println!("cargo:rerun-if-changed={}", bundle.display()); + let source = fs::read(&bundle).map_err(|_| { + "YouTube.js source bundle is missing. Initialize submodules recursively and run moon run youtubejs:bundle before Cargo." + })?; let output = PathBuf::from(env::var_os("OUT_DIR").ok_or("Cargo must set OUT_DIR")?).join("youtubei.js"); - println!("cargo:rerun-if-changed={}", output.display()); - if fs::read(&output).is_ok_and(|bytes| checksum(&bytes) == field("bundle-sha256")) { - return Ok(()); - } - - let url = format!( - "https://registry.npmjs.org/youtubei.js/-/youtubei.js-{}.tgz", - field("version") - ); - let mut archive = Vec::new(); - reqwest::blocking::Client::builder() - .timeout(Duration::from_secs(30)) - .build()? - .get(url) - .send()? - .error_for_status()? - .take(32 * 1024 * 1024 + 1) - .read_to_end(&mut archive)?; - if checksum(&archive) != field("archive-sha256") { - return Err("youtubei.js archive SHA-256 mismatch".into()); - } - let mut archive = tar::Archive::new(GzDecoder::new(archive.as_slice())); - for entry in archive.entries()? { - let entry = entry?; - if entry.path()?.as_ref() != std::path::Path::new("package/bundle/cf-worker.js") { - continue; - } - let mut bundle = Vec::new(); - entry.take(8 * 1024 * 1024 + 1).read_to_end(&mut bundle)?; - if checksum(&bundle) != field("bundle-sha256") { - return Err("youtubei.js CF-worker bundle SHA-256 mismatch".into()); - } - // Write only the verified bundle; never unpack archive paths onto disk. - fs::write(output, bundle)?; - return Ok(()); - } - Err("youtubei.js archive is missing package/bundle/cf-worker.js".into()) -} - -fn checksum(bytes: &[u8]) -> String { - format!("{:x}", Sha256::digest(bytes)) + fs::write(output, source)?; + Ok(()) } diff --git a/crates/youtubei/src/api.rs b/crates/youtubei/src/api.rs index 6f28559..5233c65 100644 --- a/crates/youtubei/src/api.rs +++ b/crates/youtubei/src/api.rs @@ -180,7 +180,7 @@ impl Session { pub async fn actions(&self) -> Result { Ok(Actions(self.0.get("actions").await?)) } - scalar_properties!(client_name: String, client_version: String, lang: String, logged_in: bool); + scalar_properties!(client_name: String, client_version: String, lang: String, logged_in: bool, user_agent: String); object_properties!(context, http, oauth); } diff --git a/moon.yml b/moon.yml index b9b6096..ec5b1bb 100644 --- a/moon.yml +++ b/moon.yml @@ -24,6 +24,12 @@ fileGroups: - .config/nextest.toml - .moon/**/*.yml - moon.yml + - vendor/youtubejs/src/**/* + - vendor/youtubejs/protos/generated/**/* + - vendor/youtubejs/package.json + - vendor/youtubejs/package-lock.json + - vendor/youtubejs/tsconfig.json + - vendor/moon.yml taskOptions: cache: true tasks: @@ -42,7 +48,7 @@ tasks: .cache/native-ffmpeg outputs: [.cache/ffmpeg/**/*] prerequisites: - deps: [~:native-ffmpeg, ~:codegen] + deps: [~:native-ffmpeg, ~:codegen, youtubejs:bundle] inputs: [] options: internal: true diff --git a/vendor/moon.yml b/vendor/moon.yml new file mode 100644 index 0000000..0b60927 --- /dev/null +++ b/vendor/moon.yml @@ -0,0 +1,37 @@ +language: typescript +layer: library +fileGroups: + sources: + - youtubejs/src/**/* + - youtubejs/protos/generated/**/* + - youtubejs/package.json + - youtubejs/package-lock.json + - youtubejs/tsconfig.json + - moon.yml +tasks: + install: + command: aube -C youtubejs install --frozen-lockfile --ignore-scripts + inputs: [youtubejs/package.json, youtubejs/package-lock.json] + options: + cache: false + internal: true + bundle: + deps: [~:install] + command: aube -C youtubejs exec --no-install esbuild + args: + - src/platform/cf-worker.ts + - --bundle + - --target=es2020 + - --keep-names + - --minify + - --format=esm + - --define:global=globalThis + - --conditions=module + - --outfile=../bundle/cf-worker.js + - --platform=node + inputs: ['@group(sources)'] + outputs: [bundle/cf-worker.js] + test: + deps: [~:bundle] + command: aube -C youtubejs exec --no-install vitest run --root .. --globals --testTimeout 30000 tests/youtubejs.test.js + inputs: ['@group(sources)', tests/youtubejs.test.js] diff --git a/vendor/tests/youtubejs.test.js b/vendor/tests/youtubejs.test.js new file mode 100644 index 0000000..da34e29 --- /dev/null +++ b/vendor/tests/youtubejs.test.js @@ -0,0 +1,34 @@ +import { Innertube, YTNodes } from '../bundle/cf-worker.js'; + +describe('embedded YouTube.js', () => { + test.each([false, true])('session accepts optional install data: %s', async (withInstallData) => { + const device = Array(108).fill(null); + device[0] = 'en'; + device[1] = 'US'; + device[13] = 'synthetic-visitor'; + device[16] = '2.20260925.01.00'; + if (withInstallData) device[61] = ['synthetic-install']; + let requests = 0; + const youtube = await Innertube.create({ + generate_session_locally: false, + retrieve_player: false, + retrieve_innertube_config: false, + enable_session_cache: false, + fail_fast: true, + fetch: async (input) => { + expect(String(input)).toEqual('https://www.youtube.com/sw.js_data'); + requests++; + return new Response(")]}'\n" + JSON.stringify([[null, null, [[device], 'synthetic-key']]])); + } + }); + expect(requests).toEqual(1); + expect(youtube.session.context.client.visitorData).toEqual('synthetic-visitor'); + expect(youtube.session.context.client.configInfo?.appInstallData).toEqual(withInstallData ? 'synthetic-install' : undefined); + }); + + test('signed-in playlist preferences parse their text label', () => { + const field = new YTNodes.ToggleFormField({ label: { simpleText: 'Show unavailable videos' }, toggled: true }); + expect(field.label.toString()).toEqual('Show unavailable videos'); + expect(field.toggled).toEqual(true); + }); +}); diff --git a/vendor/youtubejs b/vendor/youtubejs new file mode 160000 index 0000000..600a8cb --- /dev/null +++ b/vendor/youtubejs @@ -0,0 +1 @@ +Subproject commit 600a8cb83ecb5ebd55f7b4723c17e03d3ad19498