diff --git a/dependencies.lock b/dependencies.lock index bfaf436..ea68997 100644 --- a/dependencies.lock +++ b/dependencies.lock @@ -1,6 +1,6 @@ dependencies: espp/adxl345: - component_hash: d38e681caf2cc07a170b9da4661c1eeac1337af757ff3429cff5ae238cad2296 + component_hash: bd0fb1433a046704b5213b7ed52220d6393ee12b0d5114d04dce33897187bfdf dependencies: - name: espp/base_peripheral registry_url: https://components.espressif.com @@ -12,9 +12,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.1 espp/base_component: - component_hash: 4eb622f2705843fc76215d51d834ee7297522e883378e706a998d4fa6e49f231 + component_hash: 753dd0037b7dcccb859480dda36eaf7ca83a3f30c466da7c60b957ac10356c8e dependencies: - name: espp/logger registry_url: https://components.espressif.com @@ -26,9 +26,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.1 espp/base_peripheral: - component_hash: d852ab634677571e03f7ab3e871260601c33d46b00c59c56f1361a764734fff6 + component_hash: a6fc75fd03e90382a54cb87e45e720032fed440468111975fe596eb549f05eb9 dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -40,9 +40,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.1 espp/byte90: - component_hash: 2fb7520775f6e61fc9f1003eda41c8fc3829c087d963178d63f8cd3876bc2a2c + component_hash: 17afd7977353bf0136ef208e2d1b6f122a861b7d3f070e9c590b53f6d4386789 dependencies: - name: espp/adxl345 registry_url: https://components.espressif.com @@ -68,6 +68,10 @@ dependencies: registry_url: https://components.espressif.com require: private version: '>=1.0' + - name: espp/spi + registry_url: https://components.espressif.com + require: private + version: '>=1.0' - name: espp/task registry_url: https://components.espressif.com require: private @@ -80,9 +84,9 @@ dependencies: type: service targets: - esp32s3 - version: 1.0.30 + version: 1.1.1 espp/cli: - component_hash: 3578f0bdeef0074284aee15d2dc3f3f6d8a36cd198d0c6e39a8c6cc9ebe3203e + component_hash: 129d81da026d387a5bade014e44184549c9d1af553f9042d74aca68b8a60f79a dependencies: - name: espp/logger registry_url: https://components.espressif.com @@ -94,9 +98,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.1 espp/codec: - component_hash: 5584e25d97d585cd8406a5cb121af1b74e37c0df680d4cbf697cdcf20f93e19b + component_hash: af041b491aaeaddeb7339767f5a3066c2623b2d41142fec0882e08153b2211e0 dependencies: - name: idf require: private @@ -104,9 +108,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.1 espp/cst816: - component_hash: 64eceda8866977cdf0d857b36ad841baf098c9c8086cfcf60a31af7ac3a60c3d + component_hash: 586cdae4f350b0852d0e2eafcfcd073d76076ef4ce7a2be0d25d80d3b25d9e7a dependencies: - name: espp/base_peripheral registry_url: https://components.espressif.com @@ -118,9 +122,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.1 espp/display: - component_hash: acc5fd1f5209ef98c090f1f0c6ea7eff66fd6f391474567c4d4c9132ea72d9eb + component_hash: 9e587a755703b408ad1e8c919f0734291aad2242d61e7f5c6be894a1bb8560b2 dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -144,9 +148,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.1 espp/display_drivers: - component_hash: 4cdbed44584e112a7fe154020579f682263dff382d87f5ec75a906b1437c81be + component_hash: 8f7c6601f867753835f70ef5ec5b504f8227684bf523fa0d93f4109e9623e24a dependencies: - name: espp/display registry_url: https://components.espressif.com @@ -156,15 +160,19 @@ dependencies: registry_url: https://components.espressif.com require: private version: '>=1.0' + - name: espp/spi + registry_url: https://components.espressif.com + require: private + version: '>=1.0' - name: idf require: private version: '>=5.0' source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.1 espp/esp-box: - component_hash: 6b609c8e93679786b61592365087ee7d9d51f4ecc1b495db1f3b6dc09460c9b8 + component_hash: cc9ad98fb8369cffc7b8bf13f7fd4da4998e67bb67c634177263d235e002ac60 dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -202,6 +210,10 @@ dependencies: registry_url: https://components.espressif.com require: private version: '>=1.0' + - name: espp/spi + registry_url: https://components.espressif.com + require: private + version: '>=1.0' - name: espp/task registry_url: https://components.espressif.com require: private @@ -218,9 +230,9 @@ dependencies: type: service targets: - esp32s3 - version: 1.0.30 + version: 1.1.1 espp/format: - component_hash: a36e56d8620d28997f37a41f005bd0af70ccf025f38320738f148e2c9579f2f1 + component_hash: 306438454fb4391109c50ddb43695b8405bbd339c6e219b83694d29e67e41a04 dependencies: - name: idf require: private @@ -228,9 +240,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.1 espp/gt911: - component_hash: 1eacb23d7d09bd587cb78e7cf5aa71713c1902e10f5db393daf0a068b6399b64 + component_hash: 5811399fe78e380344d6b15f143f38b1c504521ac2acbfe82477bbaff0a484c5 dependencies: - name: espp/base_peripheral registry_url: https://components.espressif.com @@ -242,9 +254,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.1 espp/i2c: - component_hash: 3a2b9d4724627cacf0fd7311ceeb1a54e52dadb555664fb4d110262bee760730 + component_hash: 38b4cbd740ad89e7d96e61f7420f48b7bd7b69a11ebfc4b66ec1ecabbeb0412b dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -264,9 +276,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.1 espp/icm42607: - component_hash: d2f226fcff1dd55b52346945decfe559bc0381cacb28ccaf2ad8a57352b43e1f + component_hash: ebe114e1250a4b35424694e7cf38c855bf216bf99ece7dee52353999cce0e0d9 dependencies: - name: espp/base_peripheral registry_url: https://components.espressif.com @@ -282,9 +294,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.1 espp/input_drivers: - component_hash: 4684a2a7e7f32ad27cd8f913f753a0f0b9b6674813aa4a142aafb079e911988c + component_hash: 35e9e5bb9485d496c99aa5bf2910e8cd49fff7b61253b508bcf91080af242458 dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -300,9 +312,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.1 espp/interrupt: - component_hash: efe025b341ab8c5f6c46b165435270c40917fab4203417c040c93733772de65e + component_hash: d104e57d02c8f9e94709354a388d564ab13d2b958bf81f473ea4e4bf5e0fdc60 dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -318,9 +330,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.1 espp/led: - component_hash: 54b46f77c1b0a99abfad0b31c5543587413f0c32f5a86f1a4f24fbe561e3bcb2 + component_hash: 4132466e1180e5d53efab93e1369133ec2ae250d874a45f163e9789566db1e5b dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -336,9 +348,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.1 espp/logger: - component_hash: 8e6fc4cb3ff9feabd726a5c6fa6465b3fafc704b7c8b283232f9bf98fa517bd8 + component_hash: 3708fb36c62257ad7f81350e70e86c50ce9f42e40b9b707c6b7e21b55b73daa5 dependencies: - name: espp/format registry_url: https://components.espressif.com @@ -350,9 +362,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.1 espp/math: - component_hash: 9fff8c6bcf2db4a715272a211a7f39738f36cad4f7bba90d03ef86429bce7fd8 + component_hash: 6ebde5c1ed8d311a9d4a1fa1c56f77dec02b7e4c958481106cb70ec8cba371f6 dependencies: - name: espp/format registry_url: https://components.espressif.com @@ -364,9 +376,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.1 espp/monitor: - component_hash: 61ec113c1a946bb6a9ad030f449aed0f632b78187ad1409ab386f853e1dea1a7 + component_hash: b80b8a54941cac69547c34a5c711d56fb32404c1a1dac89e728907096d90a418 dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -382,23 +394,23 @@ dependencies: source: registry_url: https://components.espressif.com/ type: service - version: 1.0.30 + version: 1.1.1 espp/nvs: - component_hash: 31c1eb637be8d4ab3b34b4a06b12293b545537352b5711d669402f1d8e8d9685 + component_hash: e06b4a5fc2f6c35cd7689fdded1e33bcaeb43638c6b9e2bc8548b8597b37365b dependencies: + - name: idf + require: private + version: '>=5.0' - name: espp/base_component registry_url: https://components.espressif.com require: private version: '>=1.0' - - name: idf - require: private - version: '>=5.0' source: registry_url: https://components.espressif.com/ type: service - version: 1.0.30 + version: 1.1.1 espp/pcf85063: - component_hash: 99c0862fa9cd8124c43e6b0646e35ad0678f97b48f13d211dca8f111a7ca1913 + component_hash: 4bbdbdce60d96558874db53022accfc0cb7c16bdc4ead3949bed4f0dea78885f dependencies: - name: espp/base_peripheral registry_url: https://components.espressif.com @@ -414,9 +426,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.1 espp/qmi8658: - component_hash: 6752312d63a3516355d1ceb0d342da4216dad0e5e2fb04180dbcc43d3b2cdacf + component_hash: 7c87e609b722ed4bbd66ce6215d38d857d9f2a0a9138d58863fb2b550cc3b5a0 dependencies: - name: espp/base_peripheral registry_url: https://components.espressif.com @@ -432,9 +444,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.1 espp/rtsp: - component_hash: baf4a106182c3a0a6e2251fb6e1f771be88e150bcd8109335a9f09e3728eeaa5 + component_hash: eac0533fc9811b800fd73eb0a936827bb3d5c1c76bc7eedc34e0b784e1107c83 dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -454,9 +466,9 @@ dependencies: source: registry_url: https://components.espressif.com/ type: service - version: 1.0.30 + version: 1.1.1 espp/socket: - component_hash: 61ec4c78515c373c6db4ca70c8786990abf4cfaa22f3c075374a22e2c27b7eea + component_hash: b9fa3a75f195eb43c1bb0c52ef7d5dc95dee39d9b681392e5dd9ef75baa1b3cd dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -472,9 +484,27 @@ dependencies: source: registry_url: https://components.espressif.com/ type: service - version: 1.0.30 + version: 1.1.1 + espp/spi: + component_hash: 16a2358dadaa1fbc63af3f8ea9ff38c49ed5b8f45bfe5c6e49bcefb6a757d5a9 + dependencies: + - name: espp/base_component + registry_url: https://components.espressif.com + require: private + version: '>=1.0' + - name: espp/task + registry_url: https://components.espressif.com + require: private + version: '>=1.0' + - name: idf + require: private + version: '>=5.0' + source: + registry_url: https://components.espressif.com + type: service + version: 1.1.1 espp/t-deck: - component_hash: b229c64734147f9b68f5feebb28619b41c5ed21b3480c25a91a133bef3e55a43 + component_hash: d93a5e1ed602fb2ceba41744d00b1d79f8ec080393e5be93baf5257cc9e81ece dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -504,6 +534,10 @@ dependencies: registry_url: https://components.espressif.com require: private version: '>=1.0' + - name: espp/spi + registry_url: https://components.espressif.com + require: private + version: '>=1.0' - name: espp/t_keyboard registry_url: https://components.espressif.com require: private @@ -520,9 +554,9 @@ dependencies: type: service targets: - esp32s3 - version: 1.0.30 + version: 1.1.1 espp/t_keyboard: - component_hash: 1140b12e77aaf099d6f0dedca0d1aa94f7f6e14a763d84534ad8c609d881c6c0 + component_hash: b7929c10b1bdaa53229cfb043dd0bf4d1ce54a8cc9d17c64a42586f80bf9f1f3 dependencies: - name: espp/base_peripheral registry_url: https://components.espressif.com @@ -534,9 +568,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.1 espp/task: - component_hash: cd70ed978a323cd53b25862fc9b00a01eedaff32fbca8e422ae3ea921591d4c7 + component_hash: 6ae353cd01bf89c9236a06f043ba40ee707ae71d09591c3dbbdf1bd1ae48f550 dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -548,9 +582,9 @@ dependencies: source: registry_url: https://components.espressif.com/ type: service - version: 1.0.30 + version: 1.1.1 espp/tt21100: - component_hash: 915ee0bb70d8ce70165f722e6427197d4574348856c057a535b29710e52e3495 + component_hash: 8a8893e75b48f59140adf1ab6eff35040114f5b8782d60e5c4d2b19c6ce1dafe dependencies: - name: espp/base_peripheral registry_url: https://components.espressif.com @@ -562,9 +596,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.1 espp/utils: - component_hash: d5b1c58393f34971d49e1fbd00e4429e6653d10b8ae0dc66587f8521875da723 + component_hash: c392a49fe04498463dcdc9e4d36e91003b68384b334bce486b028138be7a5552 dependencies: - name: idf require: private @@ -572,9 +606,9 @@ dependencies: source: registry_url: https://components.espressif.com type: service - version: 1.0.30 + version: 1.1.1 espp/wifi: - component_hash: 762836329b6aaa19dc343718a460859fd14222c37a676c7bf697128d0d52de97 + component_hash: 3099372d99327cca94b2857731345956fa5796dfd113bd5d14bbae2ef98e71fc dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -584,15 +618,19 @@ dependencies: registry_url: https://components.espressif.com require: private version: '>=1.0' + - name: espp/nvs + registry_url: https://components.espressif.com + require: private + version: '>=1.0' - name: idf require: private version: '>=5.0' source: registry_url: https://components.espressif.com/ type: service - version: 1.0.30 + version: 1.1.1 espp/ws-s3-touch: - component_hash: c6ff3b89b270fe8ca6f2db7df2a45aa25c0940ff6cd4ab130eb9e6aefe21cd7c + component_hash: 7a464bf7eaec860843dd26bba55ffbb9cdb798205f143f1e2a7e6c9a971c182c dependencies: - name: espp/base_component registry_url: https://components.espressif.com @@ -634,6 +672,10 @@ dependencies: registry_url: https://components.espressif.com require: private version: '>=1.0' + - name: espp/spi + registry_url: https://components.espressif.com + require: private + version: '>=1.0' - name: espp/task registry_url: https://components.espressif.com require: private @@ -646,9 +688,9 @@ dependencies: type: service targets: - esp32s3 - version: 1.0.30 + version: 1.1.1 espressif/esp-dsp: - component_hash: 42dce32d46ac93dc11f60d368e29a830e9661c7345d794b8a45c343479cae636 + component_hash: ed9ac175b14e0414399a2b33c525579414b5bad19a2a67b12ea212def96f9a98 dependencies: - name: idf require: private @@ -656,9 +698,9 @@ dependencies: source: registry_url: https://components.espressif.com/ type: service - version: 1.7.0 + version: 1.8.2 espressif/mdns: - component_hash: 3ec0af5f6bce310512e90f482388d21cc7c0e99668172d2f895356165fc6f7c5 + component_hash: 8bcf12e37c58c1d584aef32a02b92548124c7a3a9fcf548d3235c844a035e0f0 dependencies: - name: idf require: private @@ -666,18 +708,18 @@ dependencies: source: registry_url: https://components.espressif.com/ type: service - version: 1.8.2 + version: 1.11.1 idf: source: type: idf - version: 5.5.1 + version: 6.0.0 lvgl/lvgl: - component_hash: 17e68bfd21f0edf4c3ee838e2273da840bf3930e5dbc3bfa6c1190c3aed41f9f + component_hash: 184e532558c1c45fefed631f3e235423d22582aafb4630f3e8885c35281a49ae dependencies: [] source: registry_url: https://components.espressif.com type: service - version: 9.4.0 + version: 9.5.0 direct_dependencies: - espp/byte90 - espp/esp-box @@ -694,4 +736,4 @@ direct_dependencies: - idf manifest_hash: 25cebf0828f59a9658091d09c2ab43e822f281c64f36520b9f89f52b1a6fa643 target: esp32s3 -version: 2.0.0 +version: 3.0.0 diff --git a/main/main.cpp b/main/main.cpp index ad07179..4eceed9 100644 --- a/main/main.cpp +++ b/main/main.cpp @@ -1,8 +1,10 @@ #include "sdkconfig.h" #include +#include #include #include +#include #include "freertos/FreeRTOS.h" #include "freertos/task.h" @@ -50,6 +52,7 @@ static uint8_t *vram0 = nullptr; static uint8_t *vram1 = nullptr; static std::atomic num_frames_received{0}; static std::atomic num_frames_displayed{0}; +static std::atomic num_audio_frames_received{0}; static std::atomic elapsed{0}; static std::chrono::high_resolution_clock::time_point connected_time; @@ -64,14 +67,58 @@ static void push_frame(const void *frame); // rtsp std::unique_ptr start_rtsp_task; static std::shared_ptr rtsp_client; - -static constexpr int num_rows_in_vram = 50; +static std::atomic rtsp_rediscovery_requested{false}; +static std::atomic active_audio_track_id{-1}; +static std::atomic active_audio_channels{0}; +static std::atomic active_audio_sample_rate{0}; +static std::atomic active_audio_output_sample_rate{0}; +static std::atomic desired_audio_volume_percent{100.0f}; + +static constexpr int num_rows_in_vram = 16; static constexpr size_t vram_size = hal::lcd_width() * num_rows_in_vram * sizeof(hal::Pixel); static constexpr size_t fb_size = hal::lcd_width() * hal::lcd_height() * sizeof(hal::Pixel); +static constexpr size_t min_rtp_port = 10000; +static constexpr size_t max_rtp_port = 20000; static std::mutex jpeg_mutex; static std::condition_variable jpeg_cv; static constexpr size_t MAX_JPEG_FRAMES = 3; static std::deque> jpeg_frames; +static void clear_jpeg_frames(); + +#if CONFIG_HARDWARE_BOX || CONFIG_HARDWARE_TDECK +static constexpr bool bsp_supports_pcm_audio_output = true; +#else +static constexpr bool bsp_supports_pcm_audio_output = false; +#endif + +#if CONFIG_HARDWARE_BOX || CONFIG_HARDWARE_TDECK || CONFIG_HARDWARE_WS_S3_TOUCH +static constexpr bool bsp_supports_touchscreen = true; +#else +static constexpr bool bsp_supports_touchscreen = false; +#endif + +static void set_audio_output_volume(float volume_percent) { + auto clamped_volume_percent = std::clamp(volume_percent, 0.0f, 100.0f); + desired_audio_volume_percent = clamped_volume_percent; + if constexpr (bsp_supports_pcm_audio_output) { + hal::get().volume(clamped_volume_percent); + } +} + +static float get_audio_volume_for_touch_x(uint16_t touch_x) { + constexpr float max_touch_x = + hal::lcd_width() > 1 ? static_cast(hal::lcd_width() - 1) : 1.0f; + auto clamped_touch_x = std::clamp(static_cast(touch_x), 0.0f, max_touch_x); + return (clamped_touch_x / max_touch_x) * 100.0f; +} + +static std::pair get_next_rtp_port_pair() { + static std::atomic next_rtp_slot{0}; + constexpr size_t max_rtp_start_port = max_rtp_port - 1; + constexpr size_t num_rtp_slots = ((max_rtp_start_port - min_rtp_port) / 2) + 1; + size_t rtp_port = min_rtp_port + 2 * (next_rtp_slot.fetch_add(1) % num_rtp_slots); + return {rtp_port, rtp_port + 1}; +} bool start_rtsp_client(std::mutex &m, std::condition_variable &cv, bool &task_notified); int drawMCUs(JPEGDRAW *pDraw); @@ -79,6 +126,10 @@ bool display_task_fn(std::mutex &m, std::condition_variable &cv); void mdns_print_results(mdns_result_t *results); bool find_mdns_service(const char *service_name, const char *proto, std::string &host, int &port, int timeout_ms = 3000); +void reset_audio_stream_state(); +void configure_audio_playback(const espp::RtspClient &client); +void handle_rtsp_frame(int track_id, std::vector &&data); +void clear_audio_output_buffer(); extern "C" void app_main(void) { logger.info("Bootup"); @@ -120,7 +171,11 @@ extern "C" void app_main(void) { } logger.info("Allocated frame buffers: fb0 = {} B, fb1 = {} B", fb_size, fb_size); - logger.info("Allocated VRAM: vram0 = {} B, vram1 = {} B", vram_size, vram_size); + logger.info("Allocated VRAM: vram0 = {} B, vram1 = {} B ({} rows each)", vram_size, vram_size, + num_rows_in_vram); + logger.info("DMA heap after VRAM alloc: free={} B, largest_block={} B", + heap_caps_get_free_size(MALLOC_CAP_DMA), + heap_caps_get_largest_free_block(MALLOC_CAP_DMA)); // initialize the video task if (!initialize_video()) { @@ -132,6 +187,20 @@ extern "C" void app_main(void) { logger.info("Clearing screen"); clear_screen(); + if constexpr (bsp_supports_touchscreen && bsp_supports_pcm_audio_output) { + auto touch_callback = [](const auto &touch) { + auto &hw = hal::get(); + auto touchpad_data = hw.touchpad_convert(touch); + if (touchpad_data.num_touch_points == 0) { + return; + } + set_audio_output_volume(get_audio_volume_for_touch_x(touchpad_data.x)); + }; + if (!hw.initialize_touch(touch_callback)) { + logger.warn("Could not initialize touch input for audio volume control"); + } + } + // create the parsing and display task logger.info("Starting display task"); auto display_task = espp::Task::make_unique({.callback = display_task_fn, @@ -162,11 +231,11 @@ extern "C" void app_main(void) { logger.info("Disconnected from WiFi, stopping task"); // ensure the rtsp start function is not running start_rtsp_task.reset(); + rtsp_rediscovery_requested = false; + reset_audio_stream_state(); logger.info("Stopping RTSP Client"); // stop and delete the RTSP client rtsp_client.reset(); - // free mdns resources - mdns_free(); }, .on_got_ip = [](ip_event_got_ip_t *eventdata) { @@ -222,6 +291,17 @@ extern "C" void app_main(void) { out << fmt::format("Received {} frames, displayed {} frames, Framerate: {} FPS\n", num_frames_received, num_frames_displayed, num_frames_displayed / disp_elapsed); + if (auto audio_track_id = active_audio_track_id.load(); audio_track_id >= 0) { + auto input_rate = active_audio_sample_rate.load(); + auto output_rate = active_audio_output_sample_rate.load(); + out << fmt::format("Audio track {}: {} frames at {} Hz", audio_track_id, + num_audio_frames_received.load(), input_rate); + if (output_rate > 0 && output_rate != input_rate) { + out << fmt::format(" -> {} Hz", output_rate); + } + out << fmt::format(" ({} channel{})\n", active_audio_channels.load(), + active_audio_channels.load() == 1 ? "" : "s"); + } }, "Display RTSP client statistics."); @@ -235,6 +315,7 @@ extern "C" void app_main(void) { } bool start_rtsp_client(std::mutex &m, std::condition_variable &cv, bool &task_notified) { + reset_audio_stream_state(); // initialize mDNS logger.info("Initializing mDNS"); auto err = mdns_init(); @@ -249,106 +330,367 @@ bool start_rtsp_client(std::mutex &m, std::condition_variable &cv, bool &task_no err = mdns_hostname_set(hostname.c_str()); if (err != ESP_OK) { logger.error("Could not set mDNS hostname: {}", err); + mdns_free(); return true; } logger.info("mDNS hostname set to '{}'", hostname); err = mdns_instance_name_set("Camera Display"); if (err != ESP_OK) { logger.error("Could not set mDNS instance name: {}", err); + mdns_free(); return true; } - std::string mdns_service_address; - int mdns_service_port; - bool found_mdns_server{false}; - while (!found_mdns_server) { - logger.info("Searching for RTSP server..."); - found_mdns_server = - find_mdns_service("_rtsp", "_tcp", mdns_service_address, mdns_service_port, 3000); + auto wait_for_stop = [&](auto duration) { + std::unique_lock lk(m); + auto stop_requested = cv.wait_for(lk, duration, [&task_notified] { return task_notified; }); + task_notified = false; + return stop_requested; + }; + + while (!task_notified) { + std::string mdns_service_address; + int mdns_service_port = 0; + bool found_mdns_server = false; + while (!found_mdns_server && !task_notified) { + logger.info("Searching for RTSP server..."); + found_mdns_server = + find_mdns_service("_rtsp", "_tcp", mdns_service_address, mdns_service_port, 3000); + } if (task_notified) { - logger.info("Stopping RTSP client task"); - return true; + break; } - } - logger.info("Found RTSP server: {}:{}", mdns_service_address, mdns_service_port); - - // free mDNS resources - mdns_free(); - // make the rtsp client - logger.info("Starting RTSP client"); - rtsp_client = std::make_shared(espp::RtspClient::Config{ - .server_address = mdns_service_address, - .rtsp_port = mdns_service_port, - .path = "/mjpeg/1", - .on_jpeg_frame = - [](std::shared_ptr jpeg_frame) { - { - std::lock_guard lock(jpeg_mutex); - if (jpeg_frames.size() >= MAX_JPEG_FRAMES) { - jpeg_frames.pop_front(); + logger.info("Found RTSP server: {}:{}", mdns_service_address, mdns_service_port); + rtsp_rediscovery_requested = false; + + logger.info("Starting RTSP client"); + auto client = std::make_shared(espp::RtspClient::Config{ + .server_address = mdns_service_address, + .rtsp_port = mdns_service_port, + .path = "/mjpeg/1", + .on_frame = handle_rtsp_frame, + .on_jpeg_frame = + [](std::shared_ptr jpeg_frame) { + { + std::lock_guard lock(jpeg_mutex); + if (jpeg_frames.size() >= MAX_JPEG_FRAMES) { + jpeg_frames.pop_front(); + } + jpeg_frames.push_back(std::move(jpeg_frame)); } - jpeg_frames.push_back(std::move(jpeg_frame)); - } - jpeg_cv.notify_all(); - num_frames_received += 1; - }, - .log_level = espp::Logger::Verbosity::ERROR, - }); + jpeg_cv.notify_all(); + if (num_frames_received.fetch_add(1) == 0) { + logger.info("Received first RTSP JPEG frame"); + } + }, + .on_connection_lost = + []() { + logger.warn("RTSP server disappeared, returning to discovery"); + rtsp_rediscovery_requested = true; + }, + .log_level = espp::Logger::Verbosity::ERROR, + }); + rtsp_client = client; + + std::error_code ec; + for (int connect_attempt = 1; connect_attempt <= 3; connect_attempt++) { + client->connect(ec); + if (!ec) { + break; + } + logger.error("Error connecting to server (attempt {}/3): {}", connect_attempt, ec.message()); + if (connect_attempt == 3) { + break; + } + if (wait_for_stop(1s)) { + task_notified = true; + break; + } + ec.clear(); + } + if (!ec) { + client->describe(ec); + if (!ec) { + configure_audio_playback(*client); + } + } + if (!ec) { + auto [rtp_port, rtcp_port] = get_next_rtp_port_pair(); + logger.info("Using RTP/RTCP client ports {}-{}", rtp_port, rtcp_port); + client->setup(rtp_port, rtcp_port, 5s, ec); + } + if (!ec) { + client->play(ec); + } - std::error_code ec; - do { - // clear the error code - ec.clear(); - rtsp_client->connect(ec); if (ec) { - logger.error("Error connecting to server: {}", ec.message()); - logger.info("Retrying in 1s..."); - std::unique_lock lk(m); - auto stop_requested = cv.wait_for(lk, 1s, [&task_notified] { return task_notified; }); + logger.error("RTSP startup failed: {}", ec.message()); + } else { + connected_time = std::chrono::high_resolution_clock::now(); + logger.info("RTSP playback started"); + bool stop_requested = false; + while (!task_notified && !rtsp_rediscovery_requested.load()) { + if (wait_for_stop(500ms)) { + stop_requested = true; + break; + } + } if (stop_requested) { - logger.info("Stopping RTSP client task"); - return true; // exit the task if stop was requested + break; } } - } while (ec); - rtsp_client->describe(ec); - if (ec) { - logger.error("Error describing server: {}", ec.message()); + std::error_code disconnect_ec; + client->disconnect(disconnect_ec); + rtsp_client.reset(); + clear_jpeg_frames(); + clear_screen(); + reset_audio_stream_state(); + + if (task_notified) { + break; + } + if (rtsp_rediscovery_requested.exchange(false)) { + logger.info("Re-entering RTSP discovery"); + continue; + } + + logger.info("Retrying RTSP discovery in 1s..."); + if (wait_for_stop(1s)) { + break; + } } - if (task_notified) { - logger.info("Stopping RTSP client task"); - return true; // exit the task if stop was requested + + rtsp_client.reset(); + clear_jpeg_frames(); + reset_audio_stream_state(); + mdns_free(); + logger.info("Stopping RTSP client task"); + return true; +} + +void reset_audio_stream_state() { + active_audio_track_id = -1; + active_audio_channels = 0; + active_audio_sample_rate = 0; + active_audio_output_sample_rate = 0; + num_audio_frames_received = 0; + clear_audio_output_buffer(); +} + +#if CONFIG_HARDWARE_BOX || CONFIG_HARDWARE_TDECK +void clear_audio_output_buffer() {} + +static uint32_t get_audio_output_sample_rate(uint32_t input_sample_rate_hz) { +#if CONFIG_HARDWARE_BOX + return std::max(input_sample_rate_hz, 48000); +#else + return input_sample_rate_hz; +#endif +} + +static bool ensure_audio_output_ready(uint32_t sample_rate_hz) { + auto &hw = hal::get(); + if (!hw.initialize_sound(sample_rate_hz)) { + logger.error("Could not initialize BSP audio output"); + return false; + } + if (hw.audio_sample_rate() != sample_rate_hz) { + hw.audio_sample_rate(sample_rate_hz); } + set_audio_output_volume(desired_audio_volume_percent.load()); + clear_audio_output_buffer(); + return true; +} - rtsp_client->setup(ec); - if (ec) { - logger.error("Error setting up server: {}", ec.message()); +static void play_pcm_audio_frame(const uint8_t *data, size_t num_bytes, int channels, + uint32_t input_sample_rate_hz, uint32_t output_sample_rate_hz) { + constexpr size_t stereo_sample_frame_size = sizeof(int16_t) * 2; + auto input_frame_size = sizeof(int16_t) * channels; + if (input_frame_size == 0) { + return; + } + auto aligned_num_bytes = num_bytes - (num_bytes % input_frame_size); + if (aligned_num_bytes == 0) { + return; } - if (task_notified) { - logger.info("Stopping RTSP client task"); - return true; // exit the task if stop was requested + + if (channels < 1 || channels > 2) { + static int warned_channels = 0; + if (warned_channels != channels) { + logger.warn("Ignoring unsupported RTSP audio with {} channels", channels); + warned_channels = channels; + } + return; } - rtsp_client->play(ec); - if (ec) { - logger.error("Error playing server: {}", ec.message()); + if (input_sample_rate_hz == 0 || output_sample_rate_hz == 0) { + static bool warned_sample_rate = false; + if (!warned_sample_rate) { + logger.warn("Ignoring RTSP audio with invalid sample rate conversion {} -> {} Hz", + input_sample_rate_hz, output_sample_rate_hz); + warned_sample_rate = true; + } + return; } - connected_time = std::chrono::high_resolution_clock::now(); + auto *src = reinterpret_cast(data); + auto input_frames = aligned_num_bytes / input_frame_size; + if (input_frames == 0) { + return; + } - return true; // we're done with our work, no need to run again, so stop the task + static thread_local std::vector stereo_frame; + auto make_stereo_frame = [&](size_t output_frames) -> int16_t * { + stereo_frame.resize(output_frames * stereo_sample_frame_size); + return reinterpret_cast(stereo_frame.data()); + }; + + if (input_sample_rate_hz == output_sample_rate_hz) { + if (channels == 2) { + auto stereo_aligned_num_bytes = + aligned_num_bytes - (aligned_num_bytes % stereo_sample_frame_size); + if (stereo_aligned_num_bytes > 0) { + hal::get().play_audio(data, stereo_aligned_num_bytes); + } + return; + } + + auto *dst = make_stereo_frame(input_frames); + for (size_t i = 0; i < input_frames; ++i) { + dst[2 * i] = src[i]; + dst[2 * i + 1] = src[i]; + } + hal::get().play_audio(stereo_frame.data(), stereo_frame.size()); + return; + } + + size_t output_frames = + std::max(1, (static_cast(input_frames) * output_sample_rate_hz + + (input_sample_rate_hz / 2)) / + input_sample_rate_hz); + auto *dst = make_stereo_frame(output_frames); + + for (size_t out_index = 0; out_index < output_frames; ++out_index) { + uint64_t scaled_position = static_cast(out_index) * input_sample_rate_hz; + size_t base_index = std::min(scaled_position / output_sample_rate_hz, input_frames - 1); + size_t next_index = std::min(base_index + 1, input_frames - 1); + uint32_t fraction = scaled_position % output_sample_rate_hz; + + auto interpolate_channel = [&](int channel) -> int16_t { + int src_channel = std::min(channel, channels - 1); + int32_t sample0 = src[base_index * channels + src_channel]; + int32_t sample1 = src[next_index * channels + src_channel]; + int64_t delta = static_cast(sample1) - sample0; + int64_t interpolated = + static_cast(sample0) + + ((delta * fraction + (output_sample_rate_hz / 2)) / output_sample_rate_hz); + return static_cast(std::clamp(interpolated, INT16_MIN, INT16_MAX)); + }; + + dst[2 * out_index] = interpolate_channel(0); + dst[2 * out_index + 1] = interpolate_channel(channels > 1 ? 1 : 0); + } + + hal::get().play_audio(stereo_frame.data(), stereo_frame.size()); +} + +static void play_pcm_audio_frame(const uint8_t *data, size_t num_bytes, int channels) { + auto input_sample_rate_hz = static_cast(std::max(active_audio_sample_rate.load(), 0)); + auto output_sample_rate_hz = + static_cast(std::max(active_audio_output_sample_rate.load(), 0)); + play_pcm_audio_frame(data, num_bytes, channels, input_sample_rate_hz, output_sample_rate_hz); +} + +static void configure_audio_output(uint32_t input_sample_rate_hz) { + auto output_sample_rate_hz = get_audio_output_sample_rate(input_sample_rate_hz); + if (!ensure_audio_output_ready(output_sample_rate_hz)) { + return; + } + active_audio_output_sample_rate = static_cast(output_sample_rate_hz); + logger.info("Configured audio output at {} Hz for {} Hz input", output_sample_rate_hz, + input_sample_rate_hz); +} +#else +void clear_audio_output_buffer() {} +#endif + +void configure_audio_playback(const espp::RtspClient &client) { + reset_audio_stream_state(); + + const auto &tracks = client.tracks(); + auto audio_track_it = std::find_if(tracks.begin(), tracks.end(), + [](const auto &track) { return track.media_type == "audio"; }); + if (audio_track_it == tracks.end()) { + logger.info("RTSP session has no audio track"); + return; + } + + if (audio_track_it->encoding_name != "L16") { + logger.warn("RTSP audio track {} uses unsupported encoding '{}'", audio_track_it->track_id, + audio_track_it->encoding_name); + return; + } + if (audio_track_it->clock_rate <= 0) { + logger.warn("RTSP audio track {} did not advertise a valid sample rate", + audio_track_it->track_id); + return; + } + + auto channels = std::max(audio_track_it->channels, 1); + if (channels > 2) { + logger.warn("RTSP audio track {} advertises unsupported channel count {}", audio_track_it->track_id, + channels); + return; + } + + if constexpr (bsp_supports_pcm_audio_output) { +#if CONFIG_HARDWARE_BOX || CONFIG_HARDWARE_TDECK + configure_audio_output(audio_track_it->clock_rate); + if (active_audio_output_sample_rate.load() <= 0) { + return; + } +#endif + active_audio_track_id = audio_track_it->track_id; + active_audio_channels = channels; + active_audio_sample_rate = audio_track_it->clock_rate; + auto output_rate = active_audio_output_sample_rate.load(); + logger.info("Configured RTSP audio playback: track {}, {} Hz{}, {} channel{}", + audio_track_it->track_id, audio_track_it->clock_rate, + output_rate > 0 && output_rate != audio_track_it->clock_rate + ? fmt::format(" -> {} Hz", output_rate) + : "", + channels, channels == 1 ? "" : "s"); + } else { + logger.info( + "RTSP session includes audio track {}, but the selected BSP has no PCM audio output", + audio_track_it->track_id); + } +} + +void handle_rtsp_frame(int track_id, std::vector &&data) { + auto audio_track_id = active_audio_track_id.load(); + if (audio_track_id < 0 || track_id != audio_track_id || data.empty()) { + return; + } + +#if CONFIG_HARDWARE_BOX || CONFIG_HARDWARE_TDECK + play_pcm_audio_frame(data.data(), data.size(), std::max(active_audio_channels.load(), 1)); +#endif + if (num_audio_frames_received.fetch_add(1) == 0) { + logger.info("Received first RTSP audio frame"); + } } static size_t frame_buffer_index = 0; // function for drawing the minimum compressible units // cppcheck-suppress constParameterCallback int drawMCUs(JPEGDRAW *pDraw) { - int iCount = pDraw->iWidth * pDraw->iHeight; + // int iCount = pDraw->iWidth * pDraw->iHeight; auto xs = pDraw->x; auto ys = pDraw->y; - auto xe = pDraw->x + pDraw->iWidth - 1; - auto ye = pDraw->y + pDraw->iHeight - 1; + // auto xe = pDraw->x + pDraw->iWidth - 1; + // auto ye = pDraw->y + pDraw->iHeight - 1; uint16_t *dst = (uint16_t *)(frame_buffer_index ? fb1 : fb0); uint16_t *src = (uint16_t *)(pDraw->pPixels); @@ -391,9 +733,11 @@ bool display_task_fn(std::mutex &m, std::condition_variable &cv) { jpeg.setPixelType(RGB565_BIG_ENDIAN); // decode the JPEG image if (!jpeg.decode(0, 0, JPEG_USES_DMA)) { - logger.debug("Error decoding"); + logger.warn("Error decoding JPEG frame"); } else { - num_frames_displayed += 1; + if (num_frames_displayed.fetch_add(1) == 0) { + logger.info("Displayed first RTSP frame"); + } // push the frame for rendering push_frame(frame_buffer_index ? fb1 : fb0); } @@ -513,6 +857,11 @@ bool initialize_video() { return true; } +void clear_jpeg_frames() { + std::lock_guard lock(jpeg_mutex); + jpeg_frames.clear(); +} + void clear_screen() { static int buffer = 0; xQueueSend(video_queue_, &buffer, portMAX_DELAY); @@ -534,7 +883,8 @@ bool video_task_callback(std::mutex &m, std::condition_variable &cv, bool &task_ auto lcd_width = hw.lcd_width(); int x_offset = 0; int y_offset = 0; - DisplayDriver::get_offset(x_offset, y_offset); + static auto &display_driver = hw.display_driver(); + display_driver->get_offset(x_offset, y_offset); static uint16_t vram_index = 0; // has to be static so that it persists between calls diff --git a/sdkconfig.defaults b/sdkconfig.defaults index 472517f..15d8e09 100644 --- a/sdkconfig.defaults +++ b/sdkconfig.defaults @@ -75,6 +75,3 @@ CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv" # CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ_240=y CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ=240 - -# the cli library requires exceptions right now... -CONFIG_COMPILER_CXX_EXCEPTIONS=y