From e6342e02d4366ee5d323ce181cbd97bd346d2842 Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Fri, 5 Jun 2026 09:01:25 -0500 Subject: [PATCH 1/8] update deps; remove exception requirement --- dependencies.lock | 186 +++++++++++++++++++++++++++------------------ sdkconfig.defaults | 3 - 2 files changed, 114 insertions(+), 75 deletions(-) diff --git a/dependencies.lock b/dependencies.lock index bfaf436..79e1535 100644 --- a/dependencies.lock +++ b/dependencies.lock @@ -1,6 +1,6 @@ dependencies: espp/adxl345: - component_hash: d38e681caf2cc07a170b9da4661c1eeac1337af757ff3429cff5ae238cad2296 + component_hash: e81f6c3720c530ea973de11e697b4dc6db325b2e0aa7b3607af0b3715dd25483 dependencies: - name: espp/base_peripheral registry_url: https://components.espressif.com @@ -12,9 +12,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.0 espp/base_component: - component_hash: 4eb622f2705843fc76215d51d834ee7297522e883378e706a998d4fa6e49f231 + component_hash: fe5c6b467f8896dfe159ecb276d63afd1bd2506b5755f46b2a6945eb82930402 dependencies: - name: espp/logger registry_url: https://components.espressif.com @@ -26,9 +26,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.0 espp/base_peripheral: - component_hash: d852ab634677571e03f7ab3e871260601c33d46b00c59c56f1361a764734fff6 + component_hash: 5829747f7be52667dcc74957c9757c58e9e9372982fb80059265bba80049765a dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -40,9 +40,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.0 espp/byte90: - component_hash: 2fb7520775f6e61fc9f1003eda41c8fc3829c087d963178d63f8cd3876bc2a2c + component_hash: 4b831e4222dcc0380c5bba074582a08de6e8ac9e5e379c13b13af8b8b49025b7 dependencies: - name: espp/adxl345 registry_url: https://components.espressif.com @@ -68,6 +68,10 @@ dependencies: registry_url: https://components.espressif.com require: private version: '>=1.0' + - name: espp/spi + registry_url: https://components.espressif.com + require: private + version: '>=1.0' - name: espp/task registry_url: https://components.espressif.com require: private @@ -80,9 +84,9 @@ dependencies: type: service targets: - esp32s3 - version: 1.0.30 + version: 1.1.0 espp/cli: - component_hash: 3578f0bdeef0074284aee15d2dc3f3f6d8a36cd198d0c6e39a8c6cc9ebe3203e + component_hash: 2f4bdb5e01c901011c533a829730507e8b4fd72e7227d1033d7822582b357cc5 dependencies: - name: espp/logger registry_url: https://components.espressif.com @@ -94,9 +98,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.0 espp/codec: - component_hash: 5584e25d97d585cd8406a5cb121af1b74e37c0df680d4cbf697cdcf20f93e19b + component_hash: d6d49b90125a08c71434eef9e1a68f562f040f974cf4167366ade09a7fde1144 dependencies: - name: idf require: private @@ -104,9 +108,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.0 espp/cst816: - component_hash: 64eceda8866977cdf0d857b36ad841baf098c9c8086cfcf60a31af7ac3a60c3d + component_hash: e973dbf796f96eaab26fe74dc7101291012ab40adfcc4a0ed78e390b28e5df92 dependencies: - name: espp/base_peripheral registry_url: https://components.espressif.com @@ -118,9 +122,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.0 espp/display: - component_hash: acc5fd1f5209ef98c090f1f0c6ea7eff66fd6f391474567c4d4c9132ea72d9eb + component_hash: 22951d33bedea83d8507d41d9ac143b94d86db68e8ce0f1ac3369a13b31d082e dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -144,9 +148,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.0 espp/display_drivers: - component_hash: 4cdbed44584e112a7fe154020579f682263dff382d87f5ec75a906b1437c81be + component_hash: 388c06d494c803f3b2909ed6b50f1525d6bc943153d08e5c201d874a6425f6d8 dependencies: - name: espp/display registry_url: https://components.espressif.com @@ -156,15 +160,19 @@ dependencies: registry_url: https://components.espressif.com require: private version: '>=1.0' + - name: espp/spi + registry_url: https://components.espressif.com + require: private + version: '>=1.0' - name: idf require: private version: '>=5.0' source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.0 espp/esp-box: - component_hash: 6b609c8e93679786b61592365087ee7d9d51f4ecc1b495db1f3b6dc09460c9b8 + component_hash: d3d9c8b94c7f5c364154888072cb41cf89f07b03fff89cb165e1c769108836b8 dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -202,6 +210,10 @@ dependencies: registry_url: https://components.espressif.com require: private version: '>=1.0' + - name: espp/spi + registry_url: https://components.espressif.com + require: private + version: '>=1.0' - name: espp/task registry_url: https://components.espressif.com require: private @@ -218,9 +230,9 @@ dependencies: type: service targets: - esp32s3 - version: 1.0.30 + version: 1.1.0 espp/format: - component_hash: a36e56d8620d28997f37a41f005bd0af70ccf025f38320738f148e2c9579f2f1 + component_hash: 5ad1f5d2a3cdf8f6ba37cf59faecd36f23cd91ab516e69a73aa6b6e2d6f32a05 dependencies: - name: idf require: private @@ -228,9 +240,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.0 espp/gt911: - component_hash: 1eacb23d7d09bd587cb78e7cf5aa71713c1902e10f5db393daf0a068b6399b64 + component_hash: 157e63b8c5e7d4753c7d418abe79118d96f348e1c6c32f0818d83e777d8ba135 dependencies: - name: espp/base_peripheral registry_url: https://components.espressif.com @@ -242,9 +254,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.0 espp/i2c: - component_hash: 3a2b9d4724627cacf0fd7311ceeb1a54e52dadb555664fb4d110262bee760730 + component_hash: 8321acc00b838b74b750d50f5a6b35eb617cb7caeb578b77539f4c7f25472e4a dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -264,9 +276,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.0 espp/icm42607: - component_hash: d2f226fcff1dd55b52346945decfe559bc0381cacb28ccaf2ad8a57352b43e1f + component_hash: 0f34a546b531f6e53feb3addba92c76f702f1809930770d9eafebd17d19f8cfe dependencies: - name: espp/base_peripheral registry_url: https://components.espressif.com @@ -282,9 +294,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.0 espp/input_drivers: - component_hash: 4684a2a7e7f32ad27cd8f913f753a0f0b9b6674813aa4a142aafb079e911988c + component_hash: cf39aef099ad1aafd165fee75559d399654c93c63c47293045d78440a9dc297d dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -300,9 +312,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.0 espp/interrupt: - component_hash: efe025b341ab8c5f6c46b165435270c40917fab4203417c040c93733772de65e + component_hash: 36ffbf7cf3fcbeec3318b00427fba423c2b2177f260f6d5be11928a99ad7bd59 dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -318,9 +330,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.0 espp/led: - component_hash: 54b46f77c1b0a99abfad0b31c5543587413f0c32f5a86f1a4f24fbe561e3bcb2 + component_hash: 3031397788fa4a742f3a05e0e407f3742a3f31ee670bdd7caea6b84e55ab644d dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -336,9 +348,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.0 espp/logger: - component_hash: 8e6fc4cb3ff9feabd726a5c6fa6465b3fafc704b7c8b283232f9bf98fa517bd8 + component_hash: 346715d1a1e0d5dfdf8d0078215c33f4356bc1161b46acb4068b2ca95f3bf758 dependencies: - name: espp/format registry_url: https://components.espressif.com @@ -350,9 +362,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.0 espp/math: - component_hash: 9fff8c6bcf2db4a715272a211a7f39738f36cad4f7bba90d03ef86429bce7fd8 + component_hash: 2a7851ce6b92b71e3a3b0e6b79c2233e849cf0e1d823fc67ff1e92b27ca26d9e dependencies: - name: espp/format registry_url: https://components.espressif.com @@ -364,9 +376,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.0 espp/monitor: - component_hash: 61ec113c1a946bb6a9ad030f449aed0f632b78187ad1409ab386f853e1dea1a7 + component_hash: 542a6c8797a780f8a0e31261629d365b17ea2c7a291f9feb6caefaa685d8d2f9 dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -382,9 +394,9 @@ dependencies: source: registry_url: https://components.espressif.com/ type: service - version: 1.0.30 + version: 1.1.0 espp/nvs: - component_hash: 31c1eb637be8d4ab3b34b4a06b12293b545537352b5711d669402f1d8e8d9685 + component_hash: e63d3e4f2d4c916aab7688ea2369ad49fbdf93cb0a60e5de4865e05989108bad dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -396,9 +408,9 @@ dependencies: source: registry_url: https://components.espressif.com/ type: service - version: 1.0.30 + version: 1.1.0 espp/pcf85063: - component_hash: 99c0862fa9cd8124c43e6b0646e35ad0678f97b48f13d211dca8f111a7ca1913 + component_hash: 2b18af08c54dd5548a30c56e441fc3639c95a014b76487cd620707d97efc1a35 dependencies: - name: espp/base_peripheral registry_url: https://components.espressif.com @@ -414,9 +426,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.0 espp/qmi8658: - component_hash: 6752312d63a3516355d1ceb0d342da4216dad0e5e2fb04180dbcc43d3b2cdacf + component_hash: be9105e805e19b12f1cea6eeaa9b0e006b8115a1a3934f46d7ca070930ed8074 dependencies: - name: espp/base_peripheral registry_url: https://components.espressif.com @@ -432,9 +444,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.0 espp/rtsp: - component_hash: baf4a106182c3a0a6e2251fb6e1f771be88e150bcd8109335a9f09e3728eeaa5 + component_hash: 7c73cb023875b108bd0e4943e15f5ed003650b7e6ce9db85b50b8b70dbdb6dc0 dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -454,9 +466,9 @@ dependencies: source: registry_url: https://components.espressif.com/ type: service - version: 1.0.30 + version: 1.1.0 espp/socket: - component_hash: 61ec4c78515c373c6db4ca70c8786990abf4cfaa22f3c075374a22e2c27b7eea + component_hash: e07163ff24d44f6180c99ca4af3e1394d308eaae92b30460a94bc4a504824b27 dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -472,9 +484,27 @@ dependencies: source: registry_url: https://components.espressif.com/ type: service - version: 1.0.30 + version: 1.1.0 + espp/spi: + component_hash: 0d14e18b8fadce14dfd5c901c58b4f7fcdfd4af017dd5b7ad0e436f5dafe5d72 + dependencies: + - name: espp/base_component + registry_url: https://components.espressif.com + require: private + version: '>=1.0' + - name: espp/task + registry_url: https://components.espressif.com + require: private + version: '>=1.0' + - name: idf + require: private + version: '>=5.0' + source: + registry_url: https://components.espressif.com + type: service + version: 1.1.0 espp/t-deck: - component_hash: b229c64734147f9b68f5feebb28619b41c5ed21b3480c25a91a133bef3e55a43 + component_hash: f11c69afd3d10316da4b44b0718b369729aa0273df072564f8e696fada391877 dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -504,6 +534,10 @@ dependencies: registry_url: https://components.espressif.com require: private version: '>=1.0' + - name: espp/spi + registry_url: https://components.espressif.com + require: private + version: '>=1.0' - name: espp/t_keyboard registry_url: https://components.espressif.com require: private @@ -520,9 +554,9 @@ dependencies: type: service targets: - esp32s3 - version: 1.0.30 + version: 1.1.0 espp/t_keyboard: - component_hash: 1140b12e77aaf099d6f0dedca0d1aa94f7f6e14a763d84534ad8c609d881c6c0 + component_hash: b18fa0ffb577f7b6c52547c087288045a4d4233cd426c1f833bbf2ee8a4a09c8 dependencies: - name: espp/base_peripheral registry_url: https://components.espressif.com @@ -534,9 +568,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.0 espp/task: - component_hash: cd70ed978a323cd53b25862fc9b00a01eedaff32fbca8e422ae3ea921591d4c7 + component_hash: bdb83d44983065d36dc43be198d9b9e1d2ed9390e8f3007cc7cab173b485a52c dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -548,9 +582,9 @@ dependencies: source: registry_url: https://components.espressif.com/ type: service - version: 1.0.30 + version: 1.1.0 espp/tt21100: - component_hash: 915ee0bb70d8ce70165f722e6427197d4574348856c057a535b29710e52e3495 + component_hash: 2c4e98b3c056fbbb96754a2ea2513f1d0051e4bac8de7afc774686efab83a34e dependencies: - name: espp/base_peripheral registry_url: https://components.espressif.com @@ -562,9 +596,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.0 espp/utils: - component_hash: d5b1c58393f34971d49e1fbd00e4429e6653d10b8ae0dc66587f8521875da723 + component_hash: f2dd093d8e48c516a0d06071f936f972168b9deec46c6611e595c06d1cd89620 dependencies: - name: idf require: private @@ -572,9 +606,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.0 espp/wifi: - component_hash: 762836329b6aaa19dc343718a460859fd14222c37a676c7bf697128d0d52de97 + component_hash: 00d725da9855516cb6ee8505fae39e8947504b2d661f58d28eb9c70cb01c6585 dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -584,15 +618,19 @@ dependencies: registry_url: https://components.espressif.com require: private version: '>=1.0' + - name: espp/nvs + registry_url: https://components.espressif.com + require: private + version: '>=1.0' - name: idf require: private version: '>=5.0' source: registry_url: https://components.espressif.com/ type: service - version: 1.0.30 + version: 1.1.0 espp/ws-s3-touch: - component_hash: c6ff3b89b270fe8ca6f2db7df2a45aa25c0940ff6cd4ab130eb9e6aefe21cd7c + component_hash: 0b52c5c57df805a49b2f3661c699112112c81662c19859bc899bf8563d31bf9e dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -634,6 +672,10 @@ dependencies: registry_url: https://components.espressif.com require: private version: '>=1.0' + - name: espp/spi + registry_url: https://components.espressif.com + require: private + version: '>=1.0' - name: espp/task registry_url: https://components.espressif.com require: private @@ -646,9 +688,9 @@ dependencies: type: service targets: - esp32s3 - version: 1.0.30 + version: 1.1.0 espressif/esp-dsp: - component_hash: 42dce32d46ac93dc11f60d368e29a830e9661c7345d794b8a45c343479cae636 + component_hash: ed9ac175b14e0414399a2b33c525579414b5bad19a2a67b12ea212def96f9a98 dependencies: - name: idf require: private @@ -656,9 +698,9 @@ dependencies: source: registry_url: https://components.espressif.com/ type: service - version: 1.7.0 + version: 1.8.2 espressif/mdns: - component_hash: 3ec0af5f6bce310512e90f482388d21cc7c0e99668172d2f895356165fc6f7c5 + component_hash: 8bcf12e37c58c1d584aef32a02b92548124c7a3a9fcf548d3235c844a035e0f0 dependencies: - name: idf require: private @@ -666,18 +708,18 @@ dependencies: source: registry_url: https://components.espressif.com/ type: service - version: 1.8.2 + version: 1.11.1 idf: source: type: idf - version: 5.5.1 + version: 6.0.0 lvgl/lvgl: - component_hash: 17e68bfd21f0edf4c3ee838e2273da840bf3930e5dbc3bfa6c1190c3aed41f9f + component_hash: 184e532558c1c45fefed631f3e235423d22582aafb4630f3e8885c35281a49ae dependencies: [] source: registry_url: https://components.espressif.com type: service - version: 9.4.0 + version: 9.5.0 direct_dependencies: - espp/byte90 - espp/esp-box @@ -694,4 +736,4 @@ direct_dependencies: - idf manifest_hash: 25cebf0828f59a9658091d09c2ab43e822f281c64f36520b9f89f52b1a6fa643 target: esp32s3 -version: 2.0.0 +version: 3.0.0 diff --git a/sdkconfig.defaults b/sdkconfig.defaults index 472517f..15d8e09 100644 --- a/sdkconfig.defaults +++ b/sdkconfig.defaults @@ -75,6 +75,3 @@ CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv" # CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ_240=y CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ=240 - -# the cli library requires exceptions right now... -CONFIG_COMPILER_CXX_EXCEPTIONS=y From be6d5b0e1ba394e5588ad7fd8278425a26e4faba Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Sun, 7 Jun 2026 17:25:59 -0500 Subject: [PATCH 2/8] feat(audio): Add support for audio playback if the server provides it --- main/main.cpp | 452 ++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 380 insertions(+), 72 deletions(-) diff --git a/main/main.cpp b/main/main.cpp index ad07179..75f14ff 100644 --- a/main/main.cpp +++ b/main/main.cpp @@ -1,8 +1,10 @@ #include "sdkconfig.h" #include +#include #include #include +#include #include "freertos/FreeRTOS.h" #include "freertos/task.h" @@ -50,6 +52,7 @@ static uint8_t *vram0 = nullptr; static uint8_t *vram1 = nullptr; static std::atomic num_frames_received{0}; static std::atomic num_frames_displayed{0}; +static std::atomic num_audio_frames_received{0}; static std::atomic elapsed{0}; static std::chrono::high_resolution_clock::time_point connected_time; @@ -64,14 +67,35 @@ static void push_frame(const void *frame); // rtsp std::unique_ptr start_rtsp_task; static std::shared_ptr rtsp_client; +static std::atomic rtsp_rediscovery_requested{false}; +static std::atomic active_audio_track_id{-1}; +static std::atomic active_audio_channels{0}; +static std::atomic active_audio_sample_rate{0}; +static std::atomic active_audio_output_sample_rate{0}; -static constexpr int num_rows_in_vram = 50; +static constexpr int num_rows_in_vram = 16; static constexpr size_t vram_size = hal::lcd_width() * num_rows_in_vram * sizeof(hal::Pixel); static constexpr size_t fb_size = hal::lcd_width() * hal::lcd_height() * sizeof(hal::Pixel); +static constexpr size_t min_rtp_port = 10000; +static constexpr size_t max_rtp_port = 20000; static std::mutex jpeg_mutex; static std::condition_variable jpeg_cv; static constexpr size_t MAX_JPEG_FRAMES = 3; static std::deque> jpeg_frames; +static void clear_jpeg_frames(); + +#if CONFIG_HARDWARE_BOX || CONFIG_HARDWARE_TDECK +static constexpr bool bsp_supports_pcm_audio_output = true; +#else +static constexpr bool bsp_supports_pcm_audio_output = false; +#endif + +static std::pair get_next_rtp_port_pair() { + static std::atomic next_rtp_slot{0}; + constexpr size_t num_rtp_slots = ((max_rtp_port - min_rtp_port) / 2) + 1; + size_t rtp_port = min_rtp_port + 2 * (next_rtp_slot.fetch_add(1) % num_rtp_slots); + return {rtp_port, rtp_port + 1}; +} bool start_rtsp_client(std::mutex &m, std::condition_variable &cv, bool &task_notified); int drawMCUs(JPEGDRAW *pDraw); @@ -79,6 +103,10 @@ bool display_task_fn(std::mutex &m, std::condition_variable &cv); void mdns_print_results(mdns_result_t *results); bool find_mdns_service(const char *service_name, const char *proto, std::string &host, int &port, int timeout_ms = 3000); +void reset_audio_stream_state(); +void configure_audio_playback(const espp::RtspClient &client); +void handle_rtsp_frame(int track_id, std::vector &&data); +void clear_audio_output_buffer(); extern "C" void app_main(void) { logger.info("Bootup"); @@ -120,7 +148,11 @@ extern "C" void app_main(void) { } logger.info("Allocated frame buffers: fb0 = {} B, fb1 = {} B", fb_size, fb_size); - logger.info("Allocated VRAM: vram0 = {} B, vram1 = {} B", vram_size, vram_size); + logger.info("Allocated VRAM: vram0 = {} B, vram1 = {} B ({} rows each)", vram_size, vram_size, + num_rows_in_vram); + logger.info("DMA heap after VRAM alloc: free={} B, largest_block={} B", + heap_caps_get_free_size(MALLOC_CAP_DMA), + heap_caps_get_largest_free_block(MALLOC_CAP_DMA)); // initialize the video task if (!initialize_video()) { @@ -162,11 +194,11 @@ extern "C" void app_main(void) { logger.info("Disconnected from WiFi, stopping task"); // ensure the rtsp start function is not running start_rtsp_task.reset(); + rtsp_rediscovery_requested = false; + reset_audio_stream_state(); logger.info("Stopping RTSP Client"); // stop and delete the RTSP client rtsp_client.reset(); - // free mdns resources - mdns_free(); }, .on_got_ip = [](ip_event_got_ip_t *eventdata) { @@ -222,6 +254,17 @@ extern "C" void app_main(void) { out << fmt::format("Received {} frames, displayed {} frames, Framerate: {} FPS\n", num_frames_received, num_frames_displayed, num_frames_displayed / disp_elapsed); + if (auto audio_track_id = active_audio_track_id.load(); audio_track_id >= 0) { + auto input_rate = active_audio_sample_rate.load(); + auto output_rate = active_audio_output_sample_rate.load(); + out << fmt::format("Audio track {}: {} frames at {} Hz", audio_track_id, + num_audio_frames_received.load(), input_rate); + if (output_rate > 0 && output_rate != input_rate) { + out << fmt::format(" -> {} Hz", output_rate); + } + out << fmt::format(" ({} channel{})\n", active_audio_channels.load(), + active_audio_channels.load() == 1 ? "" : "s"); + } }, "Display RTSP client statistics."); @@ -235,6 +278,7 @@ extern "C" void app_main(void) { } bool start_rtsp_client(std::mutex &m, std::condition_variable &cv, bool &task_notified) { + reset_audio_stream_state(); // initialize mDNS logger.info("Initializing mDNS"); auto err = mdns_init(); @@ -249,106 +293,362 @@ bool start_rtsp_client(std::mutex &m, std::condition_variable &cv, bool &task_no err = mdns_hostname_set(hostname.c_str()); if (err != ESP_OK) { logger.error("Could not set mDNS hostname: {}", err); + mdns_free(); return true; } logger.info("mDNS hostname set to '{}'", hostname); err = mdns_instance_name_set("Camera Display"); if (err != ESP_OK) { logger.error("Could not set mDNS instance name: {}", err); + mdns_free(); return true; } - std::string mdns_service_address; - int mdns_service_port; - bool found_mdns_server{false}; - while (!found_mdns_server) { - logger.info("Searching for RTSP server..."); - found_mdns_server = - find_mdns_service("_rtsp", "_tcp", mdns_service_address, mdns_service_port, 3000); + auto wait_for_stop = [&](auto duration) { + std::unique_lock lk(m); + auto stop_requested = cv.wait_for(lk, duration, [&task_notified] { return task_notified; }); + task_notified = false; + return stop_requested; + }; + + while (!task_notified) { + std::string mdns_service_address; + int mdns_service_port = 0; + bool found_mdns_server = false; + while (!found_mdns_server && !task_notified) { + logger.info("Searching for RTSP server..."); + found_mdns_server = + find_mdns_service("_rtsp", "_tcp", mdns_service_address, mdns_service_port, 3000); + } if (task_notified) { - logger.info("Stopping RTSP client task"); - return true; + break; } - } - logger.info("Found RTSP server: {}:{}", mdns_service_address, mdns_service_port); - - // free mDNS resources - mdns_free(); - // make the rtsp client - logger.info("Starting RTSP client"); - rtsp_client = std::make_shared(espp::RtspClient::Config{ - .server_address = mdns_service_address, - .rtsp_port = mdns_service_port, - .path = "/mjpeg/1", - .on_jpeg_frame = - [](std::shared_ptr jpeg_frame) { - { - std::lock_guard lock(jpeg_mutex); - if (jpeg_frames.size() >= MAX_JPEG_FRAMES) { - jpeg_frames.pop_front(); + logger.info("Found RTSP server: {}:{}", mdns_service_address, mdns_service_port); + rtsp_rediscovery_requested = false; + + logger.info("Starting RTSP client"); + auto client = std::make_shared(espp::RtspClient::Config{ + .server_address = mdns_service_address, + .rtsp_port = mdns_service_port, + .path = "/mjpeg/1", + .on_frame = handle_rtsp_frame, + .on_jpeg_frame = + [](std::shared_ptr jpeg_frame) { + { + std::lock_guard lock(jpeg_mutex); + if (jpeg_frames.size() >= MAX_JPEG_FRAMES) { + jpeg_frames.pop_front(); + } + jpeg_frames.push_back(std::move(jpeg_frame)); } - jpeg_frames.push_back(std::move(jpeg_frame)); - } - jpeg_cv.notify_all(); - num_frames_received += 1; - }, - .log_level = espp::Logger::Verbosity::ERROR, - }); + jpeg_cv.notify_all(); + if (num_frames_received.fetch_add(1) == 0) { + logger.info("Received first RTSP JPEG frame"); + } + }, + .on_connection_lost = + []() { + logger.warn("RTSP server disappeared, returning to discovery"); + rtsp_rediscovery_requested = true; + }, + .log_level = espp::Logger::Verbosity::ERROR, + }); + rtsp_client = client; + + std::error_code ec; + for (int connect_attempt = 1; connect_attempt <= 3; connect_attempt++) { + client->connect(ec); + if (!ec) { + break; + } + logger.error("Error connecting to server (attempt {}/3): {}", connect_attempt, ec.message()); + if (connect_attempt == 3) { + break; + } + if (wait_for_stop(1s)) { + task_notified = true; + break; + } + ec.clear(); + } + if (!ec) { + client->describe(ec); + if (!ec) { + configure_audio_playback(*client); + } + } + if (!ec) { + auto [rtp_port, rtcp_port] = get_next_rtp_port_pair(); + logger.info("Using RTP/RTCP client ports {}-{}", rtp_port, rtcp_port); + client->setup(rtp_port, rtcp_port, 5s, ec); + } + if (!ec) { + client->play(ec); + } - std::error_code ec; - do { - // clear the error code - ec.clear(); - rtsp_client->connect(ec); if (ec) { - logger.error("Error connecting to server: {}", ec.message()); - logger.info("Retrying in 1s..."); - std::unique_lock lk(m); - auto stop_requested = cv.wait_for(lk, 1s, [&task_notified] { return task_notified; }); + logger.error("RTSP startup failed: {}", ec.message()); + } else { + connected_time = std::chrono::high_resolution_clock::now(); + logger.info("RTSP playback started"); + bool stop_requested = false; + while (!task_notified && !rtsp_rediscovery_requested.load()) { + if (wait_for_stop(500ms)) { + stop_requested = true; + break; + } + } if (stop_requested) { - logger.info("Stopping RTSP client task"); - return true; // exit the task if stop was requested + break; + } + } + + std::error_code disconnect_ec; + client->disconnect(disconnect_ec); + rtsp_client.reset(); + clear_jpeg_frames(); + clear_screen(); + reset_audio_stream_state(); + + if (task_notified) { + break; + } + if (rtsp_rediscovery_requested.exchange(false)) { + logger.info("Re-entering RTSP discovery"); + continue; + } + + logger.info("Retrying RTSP discovery in 1s..."); + if (wait_for_stop(1s)) { + break; + } + } + + rtsp_client.reset(); + clear_jpeg_frames(); + reset_audio_stream_state(); + mdns_free(); + logger.info("Stopping RTSP client task"); + return true; +} + +void reset_audio_stream_state() { + active_audio_track_id = -1; + active_audio_channels = 0; + active_audio_sample_rate = 0; + active_audio_output_sample_rate = 0; + num_audio_frames_received = 0; + clear_audio_output_buffer(); +} + +#if CONFIG_HARDWARE_BOX || CONFIG_HARDWARE_TDECK +void clear_audio_output_buffer() {} + +static uint32_t get_audio_output_sample_rate(uint32_t input_sample_rate_hz) { +#if CONFIG_HARDWARE_BOX + return std::max(input_sample_rate_hz, 48000); +#else + return input_sample_rate_hz; +#endif +} + +static bool ensure_audio_output_ready(uint32_t sample_rate_hz) { + auto &hw = hal::get(); + if (!hw.initialize_sound(sample_rate_hz)) { + logger.error("Could not initialize BSP audio output"); + return false; + } + if (hw.audio_sample_rate() != sample_rate_hz) { + hw.audio_sample_rate(sample_rate_hz); + } + hw.volume(100.0f); + clear_audio_output_buffer(); + return true; +} + +static void play_pcm_audio_frame(const uint8_t *data, size_t num_bytes, int channels, + uint32_t input_sample_rate_hz, uint32_t output_sample_rate_hz) { + constexpr size_t stereo_sample_frame_size = sizeof(int16_t) * 2; + auto input_frame_size = sizeof(int16_t) * channels; + if (input_frame_size == 0) { + return; + } + auto aligned_num_bytes = num_bytes - (num_bytes % input_frame_size); + if (aligned_num_bytes == 0) { + return; + } + + if (channels < 1 || channels > 2) { + static int warned_channels = 0; + if (warned_channels != channels) { + logger.warn("Ignoring unsupported RTSP audio with {} channels", channels); + warned_channels = channels; + } + return; + } + + if (input_sample_rate_hz == 0 || output_sample_rate_hz == 0) { + static bool warned_sample_rate = false; + if (!warned_sample_rate) { + logger.warn("Ignoring RTSP audio with invalid sample rate conversion {} -> {} Hz", + input_sample_rate_hz, output_sample_rate_hz); + warned_sample_rate = true; + } + return; + } + + auto *src = reinterpret_cast(data); + auto input_frames = aligned_num_bytes / input_frame_size; + if (input_frames == 0) { + return; + } + + static thread_local std::vector stereo_frame; + auto make_stereo_frame = [&](size_t output_frames) -> int16_t * { + stereo_frame.resize(output_frames * stereo_sample_frame_size); + return reinterpret_cast(stereo_frame.data()); + }; + + if (input_sample_rate_hz == output_sample_rate_hz) { + if (channels == 2) { + auto stereo_aligned_num_bytes = + aligned_num_bytes - (aligned_num_bytes % stereo_sample_frame_size); + if (stereo_aligned_num_bytes > 0) { + hal::get().play_audio(data, stereo_aligned_num_bytes); } + return; + } + + auto *dst = make_stereo_frame(input_frames); + for (size_t i = 0; i < input_frames; ++i) { + dst[2 * i] = src[i]; + dst[2 * i + 1] = src[i]; } - } while (ec); + hal::get().play_audio(stereo_frame.data(), stereo_frame.size()); + return; + } - rtsp_client->describe(ec); - if (ec) { - logger.error("Error describing server: {}", ec.message()); + size_t output_frames = + std::max(1, (static_cast(input_frames) * output_sample_rate_hz + + (input_sample_rate_hz / 2)) / + input_sample_rate_hz); + auto *dst = make_stereo_frame(output_frames); + + for (size_t out_index = 0; out_index < output_frames; ++out_index) { + uint64_t scaled_position = static_cast(out_index) * input_sample_rate_hz; + size_t base_index = std::min(scaled_position / output_sample_rate_hz, input_frames - 1); + size_t next_index = std::min(base_index + 1, input_frames - 1); + uint32_t fraction = scaled_position % output_sample_rate_hz; + + auto interpolate_channel = [&](int channel) -> int16_t { + int src_channel = std::min(channel, channels - 1); + int32_t sample0 = src[base_index * channels + src_channel]; + int32_t sample1 = src[next_index * channels + src_channel]; + int64_t delta = static_cast(sample1) - sample0; + int64_t interpolated = + static_cast(sample0) + + ((delta * fraction + (output_sample_rate_hz / 2)) / output_sample_rate_hz); + return static_cast(std::clamp(interpolated, INT16_MIN, INT16_MAX)); + }; + + dst[2 * out_index] = interpolate_channel(0); + dst[2 * out_index + 1] = interpolate_channel(channels > 1 ? 1 : 0); } - if (task_notified) { - logger.info("Stopping RTSP client task"); - return true; // exit the task if stop was requested + + hal::get().play_audio(stereo_frame.data(), stereo_frame.size()); +} + +static void play_pcm_audio_frame(const uint8_t *data, size_t num_bytes, int channels) { + auto input_sample_rate_hz = static_cast(std::max(active_audio_sample_rate.load(), 0)); + auto output_sample_rate_hz = + static_cast(std::max(active_audio_output_sample_rate.load(), 0)); + play_pcm_audio_frame(data, num_bytes, channels, input_sample_rate_hz, output_sample_rate_hz); +} + +static void configure_audio_output(uint32_t input_sample_rate_hz) { + auto output_sample_rate_hz = get_audio_output_sample_rate(input_sample_rate_hz); + if (!ensure_audio_output_ready(output_sample_rate_hz)) { + return; } + active_audio_output_sample_rate = static_cast(output_sample_rate_hz); + logger.info("Configured audio output at {} Hz for {} Hz input", output_sample_rate_hz, + input_sample_rate_hz); +} +#else +void clear_audio_output_buffer() {} +#endif - rtsp_client->setup(ec); - if (ec) { - logger.error("Error setting up server: {}", ec.message()); +void configure_audio_playback(const espp::RtspClient &client) { + reset_audio_stream_state(); + + const auto &tracks = client.tracks(); + auto audio_track_it = std::find_if(tracks.begin(), tracks.end(), + [](const auto &track) { return track.media_type == "audio"; }); + if (audio_track_it == tracks.end()) { + logger.info("RTSP session has no audio track"); + return; } - if (task_notified) { - logger.info("Stopping RTSP client task"); - return true; // exit the task if stop was requested + + if (audio_track_it->encoding_name != "L16") { + logger.warn("RTSP audio track {} uses unsupported encoding '{}'", audio_track_it->track_id, + audio_track_it->encoding_name); + return; } + if (audio_track_it->clock_rate <= 0) { + logger.warn("RTSP audio track {} did not advertise a valid sample rate", + audio_track_it->track_id); + return; + } + + auto channels = std::max(audio_track_it->channels, 1); - rtsp_client->play(ec); - if (ec) { - logger.error("Error playing server: {}", ec.message()); + if constexpr (bsp_supports_pcm_audio_output) { +#if CONFIG_HARDWARE_BOX || CONFIG_HARDWARE_TDECK + configure_audio_output(audio_track_it->clock_rate); + if (active_audio_output_sample_rate.load() <= 0) { + return; + } +#endif + active_audio_track_id = audio_track_it->track_id; + active_audio_channels = channels; + active_audio_sample_rate = audio_track_it->clock_rate; + auto output_rate = active_audio_output_sample_rate.load(); + logger.info("Configured RTSP audio playback: track {}, {} Hz{}, {} channel{}", + audio_track_it->track_id, audio_track_it->clock_rate, + output_rate > 0 && output_rate != audio_track_it->clock_rate + ? fmt::format(" -> {} Hz", output_rate) + : "", + channels, channels == 1 ? "" : "s"); + } else { + logger.info( + "RTSP session includes audio track {}, but the selected BSP has no PCM audio output", + audio_track_it->track_id); } +} - connected_time = std::chrono::high_resolution_clock::now(); +void handle_rtsp_frame(int track_id, std::vector &&data) { + auto audio_track_id = active_audio_track_id.load(); + if (audio_track_id < 0 || track_id != audio_track_id || data.empty()) { + return; + } - return true; // we're done with our work, no need to run again, so stop the task +#if CONFIG_HARDWARE_BOX || CONFIG_HARDWARE_TDECK + play_pcm_audio_frame(data.data(), data.size(), std::max(active_audio_channels.load(), 1)); +#endif + if (num_audio_frames_received.fetch_add(1) == 0) { + logger.info("Received first RTSP audio frame"); + } } static size_t frame_buffer_index = 0; // function for drawing the minimum compressible units // cppcheck-suppress constParameterCallback int drawMCUs(JPEGDRAW *pDraw) { - int iCount = pDraw->iWidth * pDraw->iHeight; + // int iCount = pDraw->iWidth * pDraw->iHeight; auto xs = pDraw->x; auto ys = pDraw->y; - auto xe = pDraw->x + pDraw->iWidth - 1; - auto ye = pDraw->y + pDraw->iHeight - 1; + // auto xe = pDraw->x + pDraw->iWidth - 1; + // auto ye = pDraw->y + pDraw->iHeight - 1; uint16_t *dst = (uint16_t *)(frame_buffer_index ? fb1 : fb0); uint16_t *src = (uint16_t *)(pDraw->pPixels); @@ -391,9 +691,11 @@ bool display_task_fn(std::mutex &m, std::condition_variable &cv) { jpeg.setPixelType(RGB565_BIG_ENDIAN); // decode the JPEG image if (!jpeg.decode(0, 0, JPEG_USES_DMA)) { - logger.debug("Error decoding"); + logger.warn("Error decoding JPEG frame"); } else { - num_frames_displayed += 1; + if (num_frames_displayed.fetch_add(1) == 0) { + logger.info("Displayed first RTSP frame"); + } // push the frame for rendering push_frame(frame_buffer_index ? fb1 : fb0); } @@ -513,6 +815,11 @@ bool initialize_video() { return true; } +void clear_jpeg_frames() { + std::lock_guard lock(jpeg_mutex); + jpeg_frames.clear(); +} + void clear_screen() { static int buffer = 0; xQueueSend(video_queue_, &buffer, portMAX_DELAY); @@ -534,7 +841,8 @@ bool video_task_callback(std::mutex &m, std::condition_variable &cv, bool &task_ auto lcd_width = hw.lcd_width(); int x_offset = 0; int y_offset = 0; - DisplayDriver::get_offset(x_offset, y_offset); + static auto &display_driver = hw.display_driver(); + display_driver->get_offset(x_offset, y_offset); static uint16_t vram_index = 0; // has to be static so that it persists between calls From 829a2afd7774fa735e20705ab63160c0a0135403 Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Thu, 11 Jun 2026 16:55:35 -0500 Subject: [PATCH 3/8] update deps --- dependencies.lock | 138 +++++++++++++++++++++++----------------------- 1 file changed, 69 insertions(+), 69 deletions(-) diff --git a/dependencies.lock b/dependencies.lock index 79e1535..ea68997 100644 --- a/dependencies.lock +++ b/dependencies.lock @@ -1,6 +1,6 @@ dependencies: espp/adxl345: - component_hash: e81f6c3720c530ea973de11e697b4dc6db325b2e0aa7b3607af0b3715dd25483 + component_hash: bd0fb1433a046704b5213b7ed52220d6393ee12b0d5114d04dce33897187bfdf dependencies: - name: espp/base_peripheral registry_url: https://components.espressif.com @@ -12,9 +12,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.1.0 + version: 1.1.1 espp/base_component: - component_hash: fe5c6b467f8896dfe159ecb276d63afd1bd2506b5755f46b2a6945eb82930402 + component_hash: 753dd0037b7dcccb859480dda36eaf7ca83a3f30c466da7c60b957ac10356c8e dependencies: - name: espp/logger registry_url: https://components.espressif.com @@ -26,9 +26,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.1.0 + version: 1.1.1 espp/base_peripheral: - component_hash: 5829747f7be52667dcc74957c9757c58e9e9372982fb80059265bba80049765a + component_hash: a6fc75fd03e90382a54cb87e45e720032fed440468111975fe596eb549f05eb9 dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -40,9 +40,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.1.0 + version: 1.1.1 espp/byte90: - component_hash: 4b831e4222dcc0380c5bba074582a08de6e8ac9e5e379c13b13af8b8b49025b7 + component_hash: 17afd7977353bf0136ef208e2d1b6f122a861b7d3f070e9c590b53f6d4386789 dependencies: - name: espp/adxl345 registry_url: https://components.espressif.com @@ -84,9 +84,9 @@ dependencies: type: service targets: - esp32s3 - version: 1.1.0 + version: 1.1.1 espp/cli: - component_hash: 2f4bdb5e01c901011c533a829730507e8b4fd72e7227d1033d7822582b357cc5 + component_hash: 129d81da026d387a5bade014e44184549c9d1af553f9042d74aca68b8a60f79a dependencies: - name: espp/logger registry_url: https://components.espressif.com @@ -98,9 +98,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.1.0 + version: 1.1.1 espp/codec: - component_hash: d6d49b90125a08c71434eef9e1a68f562f040f974cf4167366ade09a7fde1144 + component_hash: af041b491aaeaddeb7339767f5a3066c2623b2d41142fec0882e08153b2211e0 dependencies: - name: idf require: private @@ -108,9 +108,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.1.0 + version: 1.1.1 espp/cst816: - component_hash: e973dbf796f96eaab26fe74dc7101291012ab40adfcc4a0ed78e390b28e5df92 + component_hash: 586cdae4f350b0852d0e2eafcfcd073d76076ef4ce7a2be0d25d80d3b25d9e7a dependencies: - name: espp/base_peripheral registry_url: https://components.espressif.com @@ -122,9 +122,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.1.0 + version: 1.1.1 espp/display: - component_hash: 22951d33bedea83d8507d41d9ac143b94d86db68e8ce0f1ac3369a13b31d082e + component_hash: 9e587a755703b408ad1e8c919f0734291aad2242d61e7f5c6be894a1bb8560b2 dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -148,9 +148,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.1.0 + version: 1.1.1 espp/display_drivers: - component_hash: 388c06d494c803f3b2909ed6b50f1525d6bc943153d08e5c201d874a6425f6d8 + component_hash: 8f7c6601f867753835f70ef5ec5b504f8227684bf523fa0d93f4109e9623e24a dependencies: - name: espp/display registry_url: https://components.espressif.com @@ -170,9 +170,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.1.0 + version: 1.1.1 espp/esp-box: - component_hash: d3d9c8b94c7f5c364154888072cb41cf89f07b03fff89cb165e1c769108836b8 + component_hash: cc9ad98fb8369cffc7b8bf13f7fd4da4998e67bb67c634177263d235e002ac60 dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -230,9 +230,9 @@ dependencies: type: service targets: - esp32s3 - version: 1.1.0 + version: 1.1.1 espp/format: - component_hash: 5ad1f5d2a3cdf8f6ba37cf59faecd36f23cd91ab516e69a73aa6b6e2d6f32a05 + component_hash: 306438454fb4391109c50ddb43695b8405bbd339c6e219b83694d29e67e41a04 dependencies: - name: idf require: private @@ -240,9 +240,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.1.0 + version: 1.1.1 espp/gt911: - component_hash: 157e63b8c5e7d4753c7d418abe79118d96f348e1c6c32f0818d83e777d8ba135 + component_hash: 5811399fe78e380344d6b15f143f38b1c504521ac2acbfe82477bbaff0a484c5 dependencies: - name: espp/base_peripheral registry_url: https://components.espressif.com @@ -254,9 +254,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.1.0 + version: 1.1.1 espp/i2c: - component_hash: 8321acc00b838b74b750d50f5a6b35eb617cb7caeb578b77539f4c7f25472e4a + component_hash: 38b4cbd740ad89e7d96e61f7420f48b7bd7b69a11ebfc4b66ec1ecabbeb0412b dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -276,9 +276,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.1.0 + version: 1.1.1 espp/icm42607: - component_hash: 0f34a546b531f6e53feb3addba92c76f702f1809930770d9eafebd17d19f8cfe + component_hash: ebe114e1250a4b35424694e7cf38c855bf216bf99ece7dee52353999cce0e0d9 dependencies: - name: espp/base_peripheral registry_url: https://components.espressif.com @@ -294,9 +294,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.1.0 + version: 1.1.1 espp/input_drivers: - component_hash: cf39aef099ad1aafd165fee75559d399654c93c63c47293045d78440a9dc297d + component_hash: 35e9e5bb9485d496c99aa5bf2910e8cd49fff7b61253b508bcf91080af242458 dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -312,9 +312,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.1.0 + version: 1.1.1 espp/interrupt: - component_hash: 36ffbf7cf3fcbeec3318b00427fba423c2b2177f260f6d5be11928a99ad7bd59 + component_hash: d104e57d02c8f9e94709354a388d564ab13d2b958bf81f473ea4e4bf5e0fdc60 dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -330,9 +330,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.1.0 + version: 1.1.1 espp/led: - component_hash: 3031397788fa4a742f3a05e0e407f3742a3f31ee670bdd7caea6b84e55ab644d + component_hash: 4132466e1180e5d53efab93e1369133ec2ae250d874a45f163e9789566db1e5b dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -348,9 +348,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.1.0 + version: 1.1.1 espp/logger: - component_hash: 346715d1a1e0d5dfdf8d0078215c33f4356bc1161b46acb4068b2ca95f3bf758 + component_hash: 3708fb36c62257ad7f81350e70e86c50ce9f42e40b9b707c6b7e21b55b73daa5 dependencies: - name: espp/format registry_url: https://components.espressif.com @@ -362,9 +362,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.1.0 + version: 1.1.1 espp/math: - component_hash: 2a7851ce6b92b71e3a3b0e6b79c2233e849cf0e1d823fc67ff1e92b27ca26d9e + component_hash: 6ebde5c1ed8d311a9d4a1fa1c56f77dec02b7e4c958481106cb70ec8cba371f6 dependencies: - name: espp/format registry_url: https://components.espressif.com @@ -376,9 +376,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.1.0 + version: 1.1.1 espp/monitor: - component_hash: 542a6c8797a780f8a0e31261629d365b17ea2c7a291f9feb6caefaa685d8d2f9 + component_hash: b80b8a54941cac69547c34a5c711d56fb32404c1a1dac89e728907096d90a418 dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -394,23 +394,23 @@ dependencies: source: registry_url: https://components.espressif.com/ type: service - version: 1.1.0 + version: 1.1.1 espp/nvs: - component_hash: e63d3e4f2d4c916aab7688ea2369ad49fbdf93cb0a60e5de4865e05989108bad + component_hash: e06b4a5fc2f6c35cd7689fdded1e33bcaeb43638c6b9e2bc8548b8597b37365b dependencies: + - name: idf + require: private + version: '>=5.0' - name: espp/base_component registry_url: https://components.espressif.com require: private version: '>=1.0' - - name: idf - require: private - version: '>=5.0' source: registry_url: https://components.espressif.com/ type: service - version: 1.1.0 + version: 1.1.1 espp/pcf85063: - component_hash: 2b18af08c54dd5548a30c56e441fc3639c95a014b76487cd620707d97efc1a35 + component_hash: 4bbdbdce60d96558874db53022accfc0cb7c16bdc4ead3949bed4f0dea78885f dependencies: - name: espp/base_peripheral registry_url: https://components.espressif.com @@ -426,9 +426,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.1.0 + version: 1.1.1 espp/qmi8658: - component_hash: be9105e805e19b12f1cea6eeaa9b0e006b8115a1a3934f46d7ca070930ed8074 + component_hash: 7c87e609b722ed4bbd66ce6215d38d857d9f2a0a9138d58863fb2b550cc3b5a0 dependencies: - name: espp/base_peripheral registry_url: https://components.espressif.com @@ -444,9 +444,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.1.0 + version: 1.1.1 espp/rtsp: - component_hash: 7c73cb023875b108bd0e4943e15f5ed003650b7e6ce9db85b50b8b70dbdb6dc0 + component_hash: eac0533fc9811b800fd73eb0a936827bb3d5c1c76bc7eedc34e0b784e1107c83 dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -466,9 +466,9 @@ dependencies: source: registry_url: https://components.espressif.com/ type: service - version: 1.1.0 + version: 1.1.1 espp/socket: - component_hash: e07163ff24d44f6180c99ca4af3e1394d308eaae92b30460a94bc4a504824b27 + component_hash: b9fa3a75f195eb43c1bb0c52ef7d5dc95dee39d9b681392e5dd9ef75baa1b3cd dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -484,9 +484,9 @@ dependencies: source: registry_url: https://components.espressif.com/ type: service - version: 1.1.0 + version: 1.1.1 espp/spi: - component_hash: 0d14e18b8fadce14dfd5c901c58b4f7fcdfd4af017dd5b7ad0e436f5dafe5d72 + component_hash: 16a2358dadaa1fbc63af3f8ea9ff38c49ed5b8f45bfe5c6e49bcefb6a757d5a9 dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -502,9 +502,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.1.0 + version: 1.1.1 espp/t-deck: - component_hash: f11c69afd3d10316da4b44b0718b369729aa0273df072564f8e696fada391877 + component_hash: d93a5e1ed602fb2ceba41744d00b1d79f8ec080393e5be93baf5257cc9e81ece dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -554,9 +554,9 @@ dependencies: type: service targets: - esp32s3 - version: 1.1.0 + version: 1.1.1 espp/t_keyboard: - component_hash: b18fa0ffb577f7b6c52547c087288045a4d4233cd426c1f833bbf2ee8a4a09c8 + component_hash: b7929c10b1bdaa53229cfb043dd0bf4d1ce54a8cc9d17c64a42586f80bf9f1f3 dependencies: - name: espp/base_peripheral registry_url: https://components.espressif.com @@ -568,9 +568,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.1.0 + version: 1.1.1 espp/task: - component_hash: bdb83d44983065d36dc43be198d9b9e1d2ed9390e8f3007cc7cab173b485a52c + component_hash: 6ae353cd01bf89c9236a06f043ba40ee707ae71d09591c3dbbdf1bd1ae48f550 dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -582,9 +582,9 @@ dependencies: source: registry_url: https://components.espressif.com/ type: service - version: 1.1.0 + version: 1.1.1 espp/tt21100: - component_hash: 2c4e98b3c056fbbb96754a2ea2513f1d0051e4bac8de7afc774686efab83a34e + component_hash: 8a8893e75b48f59140adf1ab6eff35040114f5b8782d60e5c4d2b19c6ce1dafe dependencies: - name: espp/base_peripheral registry_url: https://components.espressif.com @@ -596,9 +596,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.1.0 + version: 1.1.1 espp/utils: - component_hash: f2dd093d8e48c516a0d06071f936f972168b9deec46c6611e595c06d1cd89620 + component_hash: c392a49fe04498463dcdc9e4d36e91003b68384b334bce486b028138be7a5552 dependencies: - name: idf require: private @@ -606,9 +606,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.1.0 + version: 1.1.1 espp/wifi: - component_hash: 00d725da9855516cb6ee8505fae39e8947504b2d661f58d28eb9c70cb01c6585 + component_hash: 3099372d99327cca94b2857731345956fa5796dfd113bd5d14bbae2ef98e71fc dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -628,9 +628,9 @@ dependencies: source: registry_url: https://components.espressif.com/ type: service - version: 1.1.0 + version: 1.1.1 espp/ws-s3-touch: - component_hash: 0b52c5c57df805a49b2f3661c699112112c81662c19859bc899bf8563d31bf9e + component_hash: 7a464bf7eaec860843dd26bba55ffbb9cdb798205f143f1e2a7e6c9a971c182c dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -688,7 +688,7 @@ dependencies: type: service targets: - esp32s3 - version: 1.1.0 + version: 1.1.1 espressif/esp-dsp: component_hash: ed9ac175b14e0414399a2b33c525579414b5bad19a2a67b12ea212def96f9a98 dependencies: From ae97c0cdf701ed76a7e08e8bc188192cbb9a896d Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Tue, 16 Jun 2026 10:52:35 -0500 Subject: [PATCH 4/8] allow touchscreen x-coord to set volume output --- main/main.cpp | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/main/main.cpp b/main/main.cpp index 75f14ff..7e9cbac 100644 --- a/main/main.cpp +++ b/main/main.cpp @@ -72,6 +72,7 @@ static std::atomic active_audio_track_id{-1}; static std::atomic active_audio_channels{0}; static std::atomic active_audio_sample_rate{0}; static std::atomic active_audio_output_sample_rate{0}; +static std::atomic desired_audio_volume_percent{100.0f}; static constexpr int num_rows_in_vram = 16; static constexpr size_t vram_size = hal::lcd_width() * num_rows_in_vram * sizeof(hal::Pixel); @@ -90,6 +91,27 @@ static constexpr bool bsp_supports_pcm_audio_output = true; static constexpr bool bsp_supports_pcm_audio_output = false; #endif +#if CONFIG_HARDWARE_BOX || CONFIG_HARDWARE_TDECK || CONFIG_HARDWARE_WS_S3_TOUCH +static constexpr bool bsp_supports_touchscreen = true; +#else +static constexpr bool bsp_supports_touchscreen = false; +#endif + +static void set_audio_output_volume(float volume_percent) { + auto clamped_volume_percent = std::clamp(volume_percent, 0.0f, 100.0f); + desired_audio_volume_percent = clamped_volume_percent; + if constexpr (bsp_supports_pcm_audio_output) { + hal::get().volume(clamped_volume_percent); + } +} + +static float get_audio_volume_for_touch_x(uint16_t touch_x) { + constexpr float max_touch_x = + hal::lcd_width() > 1 ? static_cast(hal::lcd_width() - 1) : 1.0f; + auto clamped_touch_x = std::clamp(static_cast(touch_x), 0.0f, max_touch_x); + return (clamped_touch_x / max_touch_x) * 100.0f; +} + static std::pair get_next_rtp_port_pair() { static std::atomic next_rtp_slot{0}; constexpr size_t num_rtp_slots = ((max_rtp_port - min_rtp_port) / 2) + 1; @@ -164,6 +186,20 @@ extern "C" void app_main(void) { logger.info("Clearing screen"); clear_screen(); + if constexpr (bsp_supports_touchscreen && bsp_supports_pcm_audio_output) { + auto touch_callback = [](const auto &touch) { + auto &hw = hal::get(); + auto touchpad_data = hw.touchpad_convert(touch); + if (touchpad_data.num_touch_points == 0) { + return; + } + set_audio_output_volume(get_audio_volume_for_touch_x(touchpad_data.x)); + }; + if (!hw.initialize_touch(touch_callback)) { + logger.warn("Could not initialize touch input for audio volume control"); + } + } + // create the parsing and display task logger.info("Starting display task"); auto display_task = espp::Task::make_unique({.callback = display_task_fn, @@ -461,7 +497,7 @@ static bool ensure_audio_output_ready(uint32_t sample_rate_hz) { if (hw.audio_sample_rate() != sample_rate_hz) { hw.audio_sample_rate(sample_rate_hz); } - hw.volume(100.0f); + set_audio_output_volume(desired_audio_volume_percent.load()); clear_audio_output_buffer(); return true; } From e0d1e5a51a1719ffeef6c6c7a76fe9b0090fc302 Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Tue, 16 Jun 2026 14:46:08 -0500 Subject: [PATCH 5/8] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- main/main.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/main/main.cpp b/main/main.cpp index 7e9cbac..65f28d6 100644 --- a/main/main.cpp +++ b/main/main.cpp @@ -637,6 +637,11 @@ void configure_audio_playback(const espp::RtspClient &client) { } auto channels = std::max(audio_track_it->channels, 1); + if (channels > 2) { + logger.warn("RTSP audio track {} advertises unsupported channel count {}", audio_track_it->track_id, + channels); + return; + } if constexpr (bsp_supports_pcm_audio_output) { #if CONFIG_HARDWARE_BOX || CONFIG_HARDWARE_TDECK From 3656a0d415d9bb6ed5a9bdd3b2b0f51895d65260 Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Tue, 16 Jun 2026 14:46:44 -0500 Subject: [PATCH 6/8] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- main/main.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/main/main.cpp b/main/main.cpp index 65f28d6..4eceed9 100644 --- a/main/main.cpp +++ b/main/main.cpp @@ -114,7 +114,8 @@ static float get_audio_volume_for_touch_x(uint16_t touch_x) { static std::pair get_next_rtp_port_pair() { static std::atomic next_rtp_slot{0}; - constexpr size_t num_rtp_slots = ((max_rtp_port - min_rtp_port) / 2) + 1; + constexpr size_t max_rtp_start_port = max_rtp_port - 1; + constexpr size_t num_rtp_slots = ((max_rtp_start_port - min_rtp_port) / 2) + 1; size_t rtp_port = min_rtp_port + 2 * (next_rtp_slot.fetch_add(1) % num_rtp_slots); return {rtp_port, rtp_port + 1}; } From 09c5cf26235a7f83c2f230d2914a67c0aef9a4db Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Wed, 17 Jun 2026 21:38:24 -0500 Subject: [PATCH 7/8] clean up code some --- main/main.cpp | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/main/main.cpp b/main/main.cpp index 7e9cbac..fb05fde 100644 --- a/main/main.cpp +++ b/main/main.cpp @@ -53,7 +53,6 @@ static uint8_t *vram1 = nullptr; static std::atomic num_frames_received{0}; static std::atomic num_frames_displayed{0}; static std::atomic num_audio_frames_received{0}; -static std::atomic elapsed{0}; static std::chrono::high_resolution_clock::time_point connected_time; // video @@ -166,6 +165,8 @@ extern "C" void app_main(void) { if (vram1) { heap_caps_free(vram1); } + heap_caps_free(fb0); + heap_caps_free(fb1); return; } @@ -179,6 +180,10 @@ extern "C" void app_main(void) { // initialize the video task if (!initialize_video()) { logger.error("Could not initialize video task"); + heap_caps_free(fb0); + heap_caps_free(fb1); + heap_caps_free(vram0); + heap_caps_free(vram1); return; } @@ -425,6 +430,10 @@ bool start_rtsp_client(std::mutex &m, std::condition_variable &cv, bool &task_no if (ec) { logger.error("RTSP startup failed: {}", ec.message()); } else { + // reset the per-session frame counters so the stats command reports the + // framerate for the current session rather than across all reconnects + num_frames_received = 0; + num_frames_displayed = 0; connected_time = std::chrono::high_resolution_clock::now(); logger.info("RTSP playback started"); bool stop_requested = false; @@ -715,7 +724,6 @@ bool display_task_fn(std::mutex &m, std::condition_variable &cv) { image = std::move(jpeg_frames.front()); jpeg_frames.pop_front(); } - static auto start = std::chrono::high_resolution_clock::now(); auto image_data = image->get_data(); logger.info("Decoding image of size {} B, shape = {} x {}", image_data.size(), image->get_width(), image->get_height()); @@ -738,8 +746,6 @@ bool display_task_fn(std::mutex &m, std::condition_variable &cv) { } else { logger.error("error opening jpeg image"); } - auto end = std::chrono::high_resolution_clock::now(); - elapsed = std::chrono::duration(end - start).count(); // signal that we do not want to stop the task return false; } @@ -808,7 +814,7 @@ bool find_mdns_service(const char *service_name, const char *proto, std::string mdns_ip_addr_t *addr = r->addr; while (addr) { if (addr->addr.type == ESP_IPADDR_TYPE_V6) { - host = fmt::format("{:02x}:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}", + host = fmt::format("{:04x}:{:04x}:{:04x}:{:04x}:{:04x}:{:04x}:{:04x}:{:04x}", IPV62STR(addr->addr.u_addr.ip6)); break; } else if (addr->addr.type == ESP_IPADDR_TYPE_V4) { @@ -840,7 +846,7 @@ bool initialize_video() { return true; } - video_queue_ = xQueueCreate(1, sizeof(uint16_t *)); + video_queue_ = xQueueCreate(1, sizeof(const void *)); using namespace std::placeholders; video_task_ = espp::Task::make_unique({ .callback = std::bind(video_task_callback, _1, _2, _3), @@ -857,11 +863,14 @@ void clear_jpeg_frames() { } void clear_screen() { - static int buffer = 0; - xQueueSend(video_queue_, &buffer, portMAX_DELAY); + // a null frame pointer signals the video task to blank the screen + const void *null_frame = nullptr; + xQueueSend(video_queue_, &null_frame, portMAX_DELAY); } -void IRAM_ATTR push_frame(const void *frame) { xQueueSend(video_queue_, &frame, portMAX_DELAY); } +static void IRAM_ATTR push_frame(const void *frame) { + xQueueSend(video_queue_, &frame, portMAX_DELAY); +} bool video_task_callback(std::mutex &m, std::condition_variable &cv, bool &task_notified) { const void *_frame_ptr; @@ -885,7 +894,7 @@ bool video_task_callback(std::mutex &m, std::condition_variable &cv, bool &task_ // special case: if _frame_ptr is null, then we simply fill the screen with 0 if (_frame_ptr == nullptr) { for (int y = 0; y < lcd_height; y += num_lines_to_write) { - Pixel *_buf = (Pixel *)((uint32_t)vram0 * (vram_index ^ 0x01) + (uint32_t)vram1 * vram_index); + Pixel *_buf = (Pixel *)(vram_index ? vram1 : vram0); int num_lines = std::min(num_lines_to_write, lcd_height - y); // memset the buffer to 0 memset(_buf, 0, lcd_width * num_lines * sizeof(Pixel)); @@ -899,8 +908,7 @@ bool video_task_callback(std::mutex &m, std::condition_variable &cv, bool &task_ } for (int y = 0; y < lcd_height; y += num_lines_to_write) { - uint16_t *_buf = - (uint16_t *)((uint32_t)vram0 * (vram_index ^ 0x01) + (uint32_t)vram1 * vram_index); + uint16_t *_buf = (uint16_t *)(vram_index ? vram1 : vram0); int num_lines = std::min(num_lines_to_write, lcd_height - y); const uint16_t *_frame = (const uint16_t *)_frame_ptr; for (int i = 0; i < num_lines; i++) { From 2898bbdd37384d2cd4bd589d4b18bec1c4f5fb7f Mon Sep 17 00:00:00 2001 From: William Emfinger Date: Wed, 17 Jun 2026 21:43:54 -0500 Subject: [PATCH 8/8] fix formatting --- main/main.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main/main.cpp b/main/main.cpp index 16c57e2..d72f4cd 100644 --- a/main/main.cpp +++ b/main/main.cpp @@ -648,8 +648,8 @@ void configure_audio_playback(const espp::RtspClient &client) { auto channels = std::max(audio_track_it->channels, 1); if (channels > 2) { - logger.warn("RTSP audio track {} advertises unsupported channel count {}", audio_track_it->track_id, - channels); + logger.warn("RTSP audio track {} advertises unsupported channel count {}", + audio_track_it->track_id, channels); return; }